mirror of
https://github.com/pnpm/pnpm.git
synced 2026-07-18 19:52:31 -04:00
Foundation for porting pnpm's side-effects cache (item #10 of #397). This PR lands the three pieces that the actual cache read/write paths depend on: - **New `pacquet-graph-hasher` crate** porting pnpm's `@pnpm/crypto.object-hasher` (`hashObject` / `hashObjectWithoutSorting`) plus `@pnpm/deps.graph-hasher` (`calcDepState` / `calcDepGraphHash`) and `ENGINE_NAME`. Byte-for-byte parity with the JS [`object-hash@3.0.0`](https://github.com/puleos/object-hash/blob/v3.0.0/index.js) bytestream format is load-bearing — the cache key is persisted on disk and shared with pnpm. The headline parity test pins `hashObject({b:1,a:2}) == "48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ="` against the upstream fixture in [`crypto/object-hasher/test/index.ts:6`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L6). - **`VerifyResult.side_effects_maps`** in `pacquet-store-dir`. The verify path (`check_pkg_files_integrity` and `build_file_maps_from_index`) used to drop `PackageFilesIndex.side_effects` after extraction. It now applies the `added`/`deleted` overlay per cache key and surfaces a `HashMap<cache_key, FilesMap>` — the same shape pnpm uses for its `PackageFilesResponse.sideEffectsMaps`. Mirrors [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121). - **`Config.side_effects_cache`** config knob (default `true`, matching pnpm). Surfaced on the wire **only** — no consumer yet. Wires up cleanly once the build-phase gate lands in #421, with no config migration needed for downstream callers. ## Scoped narrowly This is **only the foundation**. There is no `is_built` field, no `BuildModules` skip gate, and no end-to-end test of the rebuild-skip behavior. Those land in a separate follow-up tracked in #421, which also covers the WRITE path (pacquet seeding the cache itself). The natural slice for review: graph-hasher unblocks both READ and WRITE, the verify-path surfacing is the only intrusive cross-crate change, and the config knob is forward-looking. ### Hashing scope, explicitly The `pacquet-graph-hasher` port implements only the type arms pacquet actually feeds into `hashObject` for the cache-key path: strings, objects, numbers, booleans, null, and arrays in their ordered form. `Set` / `Map` / `Date` / `Buffer` / `Array` unordered-permutation arms from [upstream `object-hash`](https://github.com/puleos/object-hash/blob/v3.0.0/index.js#L257-L389) are unimplemented — no caller in pacquet's tree feeds them in today, and the test fixtures upstream uses (`hashObject({ b: new Set([…]), a: [...] })`) hash deterministically across orderings only because they're testing the library, not exercising a real call path. Adding them can land alongside a future caller that needs them.
This commit is contained in:
Zoltan Kochan
10
pacquet/Cargo.lock
generated
10
pacquet/Cargo.lock
generated
@@ -1942,6 +1942,16 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pacquet-graph-hasher"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"pretty_assertions",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pacquet-integrated-benchmark"
|
||||
version = "0.0.0"
|
||||
|
||||
@@ -27,6 +27,7 @@ pacquet-network = { path = "crates/network" }
|
||||
pacquet-config = { path = "crates/config" }
|
||||
pacquet-executor = { path = "crates/executor" }
|
||||
pacquet-diagnostics = { path = "crates/diagnostics" }
|
||||
pacquet-graph-hasher = { path = "crates/graph-hasher" }
|
||||
pacquet-store-dir = { path = "crates/store-dir" }
|
||||
pacquet-reporter = { path = "crates/reporter" }
|
||||
|
||||
|
||||
@@ -174,6 +174,24 @@ pub struct Config {
|
||||
#[default = true]
|
||||
pub verify_store_integrity: bool,
|
||||
|
||||
/// Whether to consult the side-effects cache
|
||||
/// (`PackageFilesIndex.sideEffects`) when importing a package.
|
||||
/// Read from `pnpm-workspace.yaml`'s `sideEffectsCache` field
|
||||
/// (camelCase, optional, defaults `true`). **Not yet acted on:**
|
||||
/// the build phase doesn't currently gate the rebuild-skip on
|
||||
/// this flag, and the WRITE path (populating the cache after a
|
||||
/// postinstall) is also unimplemented. Both follow up
|
||||
/// separately — tracked in pnpm/pacquet#421.
|
||||
///
|
||||
/// Default `true`, matching pnpm's `side-effects-cache` at
|
||||
/// [`config/config/src/index.ts`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/config/config/src/index.ts).
|
||||
/// Wiring the config-source plumbing through now means
|
||||
/// downstream callers can set `sideEffectsCache: false` in
|
||||
/// `pnpm-workspace.yaml` today and have the value take effect
|
||||
/// as soon as the read-path gate lands.
|
||||
#[default = true]
|
||||
pub side_effects_cache: bool,
|
||||
|
||||
/// How many times pacquet retries a failed tarball fetch on transient
|
||||
/// errors before giving up. Mirrors pnpm's `fetchRetries` (default
|
||||
/// `2`, matching `config/config/src/index.ts`). The value is the count
|
||||
|
||||
@@ -50,6 +50,7 @@ pub struct WorkspaceSettings {
|
||||
pub strict_peer_dependencies: Option<bool>,
|
||||
pub resolve_peers_from_workspace_root: Option<bool>,
|
||||
pub verify_store_integrity: Option<bool>,
|
||||
pub side_effects_cache: Option<bool>,
|
||||
pub fetch_retries: Option<u32>,
|
||||
pub fetch_retry_factor: Option<u32>,
|
||||
pub fetch_retry_mintimeout: Option<u64>,
|
||||
@@ -141,8 +142,8 @@ impl WorkspaceSettings {
|
||||
lockfile, prefer_frozen_lockfile, lockfile_include_tarball_url,
|
||||
auto_install_peers, dedupe_peer_dependents, strict_peer_dependencies,
|
||||
resolve_peers_from_workspace_root, verify_store_integrity,
|
||||
fetch_retries, fetch_retry_factor, fetch_retry_mintimeout,
|
||||
fetch_retry_maxtimeout,
|
||||
side_effects_cache, fetch_retries, fetch_retry_factor,
|
||||
fetch_retry_mintimeout, fetch_retry_maxtimeout,
|
||||
}
|
||||
|
||||
if let Some(v) = self.modules_dir {
|
||||
|
||||
@@ -125,6 +125,23 @@ fn parses_verify_store_integrity_from_yaml_and_applies() {
|
||||
assert!(!config.verify_store_integrity, "yaml override wins");
|
||||
}
|
||||
|
||||
/// `sideEffectsCache` is the side-effects cache READ-path knob from
|
||||
/// pnpm-workspace.yaml. Same shape as `verifyStoreIntegrity`:
|
||||
/// camelCase rename + `apply_to` wiring. Parsing a yaml that flips
|
||||
/// the default-true setting to false must end up at
|
||||
/// `config.side_effects_cache == false`.
|
||||
#[test]
|
||||
fn parses_side_effects_cache_from_yaml_and_applies() {
|
||||
let yaml = "sideEffectsCache: false\n";
|
||||
let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap();
|
||||
assert_eq!(settings.side_effects_cache, Some(false));
|
||||
|
||||
let mut config = Config::new();
|
||||
assert!(config.side_effects_cache, "the default is `true` to match pnpm");
|
||||
settings.apply_to(&mut config, Path::new("/irrelevant"));
|
||||
assert!(!config.side_effects_cache, "yaml override wins");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_leaves_unset_fields_alone() {
|
||||
let yaml = "storeDir: /s\n";
|
||||
|
||||
19
pacquet/crates/graph-hasher/Cargo.toml
Normal file
19
pacquet/crates/graph-hasher/Cargo.toml
Normal file
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "pacquet-graph-hasher"
|
||||
version = "0.0.1"
|
||||
publish = false
|
||||
authors.workspace = true
|
||||
description.workspace = true
|
||||
edition.workspace = true
|
||||
homepage.workspace = true
|
||||
keywords.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64 = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
321
pacquet/crates/graph-hasher/src/dep_state.rs
Normal file
321
pacquet/crates/graph-hasher/src/dep_state.rs
Normal file
@@ -0,0 +1,321 @@
|
||||
use crate::object_hasher::hash_object;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
/// Per-node identifier carrying everything `calc_dep_state` needs to
|
||||
/// hash a snapshot. Mirrors the relevant subset of pnpm's
|
||||
/// `DepsGraphNode` at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L12-L19>.
|
||||
///
|
||||
/// `full_pkg_id` is the upstream-shaped fingerprint used as the
|
||||
/// `id` field in the recursive hash — `<pkgIdWithPatchHash>:<integrity>`
|
||||
/// for packages with an integrity (`registry` resolution),
|
||||
/// or `<pkgIdWithPatchHash>:<hashObject(resolution)>` for resolutions
|
||||
/// without one (e.g. git refs). Pacquet's caller composes this
|
||||
/// before passing it in; the hasher itself is opaque to how it was
|
||||
/// computed.
|
||||
///
|
||||
/// `children` maps alias → dep-graph key for the snapshot's
|
||||
/// children. Pacquet's natural input shape is the lockfile's
|
||||
/// `snapshots[].dependencies` + `optionalDependencies` flattened,
|
||||
/// with each value resolved to the snapshot key it points at.
|
||||
pub struct DepsGraphNode<'a, K> {
|
||||
pub full_pkg_id: &'a str,
|
||||
pub children: HashMap<&'a str, K>,
|
||||
}
|
||||
|
||||
/// Memoized per-depPath state cache. Mirrors pnpm's
|
||||
/// [`DepsStateCache`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L21-L23):
|
||||
/// the result of `hash_object` for each visited node is stashed so
|
||||
/// the recursive walk over diamond-shaped graphs stays linear.
|
||||
pub type DepsStateCache<K> = HashMap<K, String>;
|
||||
|
||||
/// Inputs to [`calc_dep_state`]. Mirrors the option bag at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L29-L33>.
|
||||
pub struct CalcDepStateOptions<'a> {
|
||||
/// Output of [`crate::engine_name()`] — the platform / arch /
|
||||
/// node version prefix. Always part of the result.
|
||||
pub engine_name: &'a str,
|
||||
/// SHA-256 hex of the patch file for this package (when present).
|
||||
/// Appended as `;patch=<hash>`.
|
||||
pub patch_file_hash: Option<&'a str>,
|
||||
/// Whether to include the recursive dep-graph hash as
|
||||
/// `;deps=<hash>`. Upstream sets this to `hasSideEffects`
|
||||
/// (i.e. `!ignoreScripts && requiresBuild`) at
|
||||
/// `building/during-install/src/index.ts:202`.
|
||||
pub include_dep_graph_hash: bool,
|
||||
}
|
||||
|
||||
/// Mirrors `calcDepState` at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L25-L44>.
|
||||
///
|
||||
/// Returns the cache key for the side-effects cache. Format:
|
||||
/// `<engine_name>[;deps=<hash>][;patch=<hash>]`. Byte-for-byte
|
||||
/// parity with pnpm is required — the key is persisted on disk and
|
||||
/// shared with pnpm.
|
||||
pub fn calc_dep_state<K>(
|
||||
graph: &HashMap<K, DepsGraphNode<'_, K>>,
|
||||
cache: &mut DepsStateCache<K>,
|
||||
dep_path: &K,
|
||||
opts: &CalcDepStateOptions<'_>,
|
||||
) -> String
|
||||
where
|
||||
K: Clone + Eq + std::hash::Hash,
|
||||
{
|
||||
let mut result = opts.engine_name.to_string();
|
||||
if opts.include_dep_graph_hash {
|
||||
let deps_hash = calc_dep_graph_hash(graph, cache, &mut HashSet::new(), dep_path);
|
||||
result.push_str(";deps=");
|
||||
result.push_str(&deps_hash);
|
||||
}
|
||||
if let Some(patch) = opts.patch_file_hash {
|
||||
result.push_str(";patch=");
|
||||
result.push_str(patch);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Recursive helper for the `deps=` portion. Mirrors
|
||||
/// `calcDepGraphHash` at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L46-L80>.
|
||||
///
|
||||
/// Hashes each node as `hashObject({ id, deps })` where `deps` is
|
||||
/// the alias→child-hash map. `parents` breaks dependency cycles —
|
||||
/// when a node would re-enter via its own ancestor, the child's
|
||||
/// contribution becomes `""` (matching upstream's "node not in
|
||||
/// graph" guard at line 55, which returns the empty string).
|
||||
fn calc_dep_graph_hash<K>(
|
||||
graph: &HashMap<K, DepsGraphNode<'_, K>>,
|
||||
cache: &mut DepsStateCache<K>,
|
||||
parents: &mut HashSet<String>,
|
||||
dep_path: &K,
|
||||
) -> String
|
||||
where
|
||||
K: Clone + Eq + std::hash::Hash,
|
||||
{
|
||||
if let Some(cached) = cache.get(dep_path) {
|
||||
return cached.clone();
|
||||
}
|
||||
let Some(node) = graph.get(dep_path) else {
|
||||
return String::new();
|
||||
};
|
||||
let mut deps_obj = serde_json::Map::new();
|
||||
if !node.children.is_empty() && !parents.contains(node.full_pkg_id) {
|
||||
// Push our `full_pkg_id` for the duration of this subtree
|
||||
// so cycles short-circuit on the second visit.
|
||||
let inserted = parents.insert(node.full_pkg_id.to_string());
|
||||
for (alias, child_key) in &node.children {
|
||||
let child_hash = calc_dep_graph_hash(graph, cache, parents, child_key);
|
||||
deps_obj.insert((*alias).to_string(), Value::String(child_hash));
|
||||
}
|
||||
if inserted {
|
||||
parents.remove(node.full_pkg_id);
|
||||
}
|
||||
}
|
||||
let hashed = hash_object(&json!({
|
||||
"id": node.full_pkg_id,
|
||||
"deps": Value::Object(deps_obj),
|
||||
}));
|
||||
cache.insert(dep_path.clone(), hashed.clone());
|
||||
cache.get(dep_path).expect("just inserted").clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CalcDepStateOptions, DepsGraphNode, calc_dep_state};
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Engine-only key (no dep graph, no patch). Pure prefix path
|
||||
/// for the cheapest cache lookup. Mirrors the "include_dep_graph_hash:
|
||||
/// false" path at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L36>.
|
||||
#[test]
|
||||
fn engine_only_key() {
|
||||
let graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
let mut cache = HashMap::new();
|
||||
let result = calc_dep_state(
|
||||
&graph,
|
||||
&mut cache,
|
||||
&"foo@1.0.0".to_string(),
|
||||
&CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: None,
|
||||
include_dep_graph_hash: false,
|
||||
},
|
||||
);
|
||||
assert_eq!(result, "darwin;arm64;node20");
|
||||
}
|
||||
|
||||
/// Patch hash gets appended as `;patch=<hash>`. Combined with
|
||||
/// the engine prefix when there's no dep graph hash. Mirrors
|
||||
/// lines 40-42 of `calcDepState`.
|
||||
#[test]
|
||||
fn patch_appended_without_dep_graph_hash() {
|
||||
let graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
let mut cache = HashMap::new();
|
||||
let result = calc_dep_state(
|
||||
&graph,
|
||||
&mut cache,
|
||||
&"foo@1.0.0".to_string(),
|
||||
&CalcDepStateOptions {
|
||||
engine_name: "linux;x64;node22",
|
||||
patch_file_hash: Some("sha256-abc"),
|
||||
include_dep_graph_hash: false,
|
||||
},
|
||||
);
|
||||
assert_eq!(result, "linux;x64;node22;patch=sha256-abc");
|
||||
}
|
||||
|
||||
/// Dep-graph hash for a leaf (no children) is `hash_object({
|
||||
/// id, deps: {} })`. Both sites that consult `deps={}` (the
|
||||
/// leaf case at `calcLeafGlobalVirtualStorePath` and the
|
||||
/// children-elided case for cycle/missing-node) must agree.
|
||||
#[test]
|
||||
fn dep_graph_hash_for_leaf_uses_id_and_empty_deps() {
|
||||
let mut graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
graph.insert(
|
||||
"leaf@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "leaf@1.0.0:sha512-leaf", children: HashMap::new() },
|
||||
);
|
||||
let mut cache = HashMap::new();
|
||||
let result = calc_dep_state(
|
||||
&graph,
|
||||
&mut cache,
|
||||
&"leaf@1.0.0".to_string(),
|
||||
&CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: None,
|
||||
include_dep_graph_hash: true,
|
||||
},
|
||||
);
|
||||
// Prefix preserved, deps= section appended.
|
||||
let parts: Vec<&str> = result.split(';').collect();
|
||||
assert!(parts.len() == 4, "expected `<plat>;<arch>;node<n>;deps=<hash>`, got {result:?}");
|
||||
assert!(parts[3].starts_with("deps="), "fourth segment must be `deps=...`: {result:?}");
|
||||
assert!(parts[3][5..].len() >= 40, "hash payload must be non-trivial: {result:?}");
|
||||
}
|
||||
|
||||
/// Memoization at the cache layer: `calc_dep_graph_hash` writes
|
||||
/// each node's hash on first visit and returns the cached
|
||||
/// value on re-visit. Two leaf nodes with the same
|
||||
/// `full_pkg_id` must agree.
|
||||
#[test]
|
||||
fn cache_makes_repeat_calls_byte_equal() {
|
||||
let mut graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
graph.insert(
|
||||
"leaf@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "leaf@1.0.0:sha512-x", children: HashMap::new() },
|
||||
);
|
||||
let mut cache = HashMap::new();
|
||||
let opts = CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: None,
|
||||
include_dep_graph_hash: true,
|
||||
};
|
||||
let a = calc_dep_state(&graph, &mut cache, &"leaf@1.0.0".to_string(), &opts);
|
||||
let b = calc_dep_state(&graph, &mut cache, &"leaf@1.0.0".to_string(), &opts);
|
||||
assert_eq!(a, b);
|
||||
assert_eq!(cache.len(), 1, "cache must hold exactly the one leaf entry");
|
||||
}
|
||||
|
||||
/// Diamond graph: root depends on a and b, both depend on c.
|
||||
/// Both alias→child entries on the root must agree on the c
|
||||
/// node's hash, and the recursion must terminate.
|
||||
#[test]
|
||||
fn diamond_graph_resolves_consistently() {
|
||||
let mut graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
let mut root_children = HashMap::new();
|
||||
root_children.insert("a", "a@1.0.0".to_string());
|
||||
root_children.insert("b", "b@1.0.0".to_string());
|
||||
graph.insert(
|
||||
"root@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "root@1.0.0:sha512-root", children: root_children },
|
||||
);
|
||||
let mut a_children = HashMap::new();
|
||||
a_children.insert("c", "c@1.0.0".to_string());
|
||||
graph.insert(
|
||||
"a@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "a@1.0.0:sha512-a", children: a_children },
|
||||
);
|
||||
let mut b_children = HashMap::new();
|
||||
b_children.insert("c", "c@1.0.0".to_string());
|
||||
graph.insert(
|
||||
"b@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "b@1.0.0:sha512-b", children: b_children },
|
||||
);
|
||||
graph.insert(
|
||||
"c@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "c@1.0.0:sha512-c", children: HashMap::new() },
|
||||
);
|
||||
let mut cache = HashMap::new();
|
||||
let result = calc_dep_state(
|
||||
&graph,
|
||||
&mut cache,
|
||||
&"root@1.0.0".to_string(),
|
||||
&CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: None,
|
||||
include_dep_graph_hash: true,
|
||||
},
|
||||
);
|
||||
// root + a + b + c = 4 cache entries.
|
||||
assert_eq!(cache.len(), 4, "expected 4 cache entries for diamond, got {cache:#?}");
|
||||
assert!(result.contains(";deps="), "result must include deps section: {result:?}");
|
||||
}
|
||||
|
||||
/// Cycle: a depends on b, b depends on a. The walk must
|
||||
/// terminate (parents-set short-circuit) and produce a stable
|
||||
/// hash. Mirrors upstream's `if (!parents.has(node.fullPkgId))`
|
||||
/// guard at line 66.
|
||||
#[test]
|
||||
fn cyclic_graph_terminates_and_is_stable() {
|
||||
let mut graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
let mut a_children = HashMap::new();
|
||||
a_children.insert("b", "b@1.0.0".to_string());
|
||||
graph.insert(
|
||||
"a@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "a@1.0.0:sha512-a", children: a_children },
|
||||
);
|
||||
let mut b_children = HashMap::new();
|
||||
b_children.insert("a", "a@1.0.0".to_string());
|
||||
graph.insert(
|
||||
"b@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "b@1.0.0:sha512-b", children: b_children },
|
||||
);
|
||||
let mut cache = HashMap::new();
|
||||
let opts = CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: None,
|
||||
include_dep_graph_hash: true,
|
||||
};
|
||||
let h1 = calc_dep_state(&graph, &mut cache, &"a@1.0.0".to_string(), &opts);
|
||||
let h2 = calc_dep_state(&graph, &mut cache, &"a@1.0.0".to_string(), &opts);
|
||||
assert_eq!(h1, h2);
|
||||
}
|
||||
|
||||
/// Both patch and dep graph hashes append in upstream's order:
|
||||
/// `<engine>;deps=<h>;patch=<h>`. Mirrors index.js:36-42.
|
||||
#[test]
|
||||
fn dep_graph_and_patch_concatenate_in_upstream_order() {
|
||||
let mut graph: HashMap<String, DepsGraphNode<'_, String>> = HashMap::new();
|
||||
graph.insert(
|
||||
"x@1.0.0".to_string(),
|
||||
DepsGraphNode { full_pkg_id: "x@1.0.0:sha512-x", children: HashMap::new() },
|
||||
);
|
||||
let mut cache = HashMap::new();
|
||||
let result = calc_dep_state(
|
||||
&graph,
|
||||
&mut cache,
|
||||
&"x@1.0.0".to_string(),
|
||||
&CalcDepStateOptions {
|
||||
engine_name: "darwin;arm64;node20",
|
||||
patch_file_hash: Some("patchhex"),
|
||||
include_dep_graph_hash: true,
|
||||
},
|
||||
);
|
||||
let deps_pos = result.find(";deps=").expect("deps section present");
|
||||
let patch_pos = result.find(";patch=").expect("patch section present");
|
||||
assert!(deps_pos < patch_pos, "deps must come before patch in {result:?}");
|
||||
}
|
||||
}
|
||||
92
pacquet/crates/graph-hasher/src/engine_name.rs
Normal file
92
pacquet/crates/graph-hasher/src/engine_name.rs
Normal file
@@ -0,0 +1,92 @@
|
||||
/// Compute pnpm's `ENGINE_NAME` string — the same value pnpm uses
|
||||
/// as the side-effects cache key prefix.
|
||||
///
|
||||
/// Ports
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/core/constants/src/index.ts#L7>:
|
||||
/// ```js
|
||||
/// `${process.platform};${process.arch};node${process.version.split('.')[0].substring(1)}`
|
||||
/// ```
|
||||
///
|
||||
/// Example outputs:
|
||||
/// - `"darwin;arm64;node20"`
|
||||
/// - `"linux;x64;node22"`
|
||||
/// - `"win32;x64;node24"`
|
||||
///
|
||||
/// `node_major` is the Node major version (e.g. `20`, `22`, `24`).
|
||||
/// Callers pass it as a number because the discovery side (spawning
|
||||
/// `node --version` or reading `npm_node_execpath`) is policy and
|
||||
/// doesn't belong in this hasher crate.
|
||||
///
|
||||
/// `platform` and `arch` default to the running host via the
|
||||
/// static `std::env::consts` constants mapped through Node's
|
||||
/// naming scheme. Production callers can pass `None` to get the
|
||||
/// host values; tests can pin both for cache-key round-trip.
|
||||
pub fn engine_name(node_major: u32, platform: Option<&str>, arch: Option<&str>) -> String {
|
||||
let platform = platform.unwrap_or_else(|| host_platform());
|
||||
let arch = arch.unwrap_or_else(|| host_arch());
|
||||
format!("{platform};{arch};node{node_major}")
|
||||
}
|
||||
|
||||
/// Map `std::env::consts::OS` to Node's `process.platform` naming.
|
||||
/// Node uses `darwin` / `linux` / `win32` / `freebsd` / `openbsd` /
|
||||
/// `sunos` / `aix` / `android`. Rust uses `macos` / `linux` /
|
||||
/// `windows` / `freebsd` / `openbsd` / `solaris` / `aix` /
|
||||
/// `android`. Only `macos`, `windows`, and `solaris` differ.
|
||||
fn host_platform() -> &'static str {
|
||||
match std::env::consts::OS {
|
||||
"macos" => "darwin",
|
||||
"windows" => "win32",
|
||||
"solaris" => "sunos",
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// Map `std::env::consts::ARCH` to Node's `process.arch` naming.
|
||||
/// Node uses `x64` / `arm64` / `ia32` / `arm` / `s390x` / `ppc64`
|
||||
/// / `ppc64` (LE, same string) / `loong64` / `riscv64`. Rust uses
|
||||
/// `x86_64` / `aarch64` / `x86` / `arm` / `s390x` / `powerpc64` /
|
||||
/// `powerpc64le` / `loongarch64` / `riscv64`. Mappings below mirror
|
||||
/// what Node itself emits on each target — anything left as
|
||||
/// passthrough (e.g. `arm`, `s390x`, `riscv64`) already matches
|
||||
/// between the two naming schemes.
|
||||
fn host_arch() -> &'static str {
|
||||
match std::env::consts::ARCH {
|
||||
"x86_64" => "x64",
|
||||
"aarch64" => "arm64",
|
||||
"x86" => "ia32",
|
||||
// Node calls big-endian and little-endian POWER both
|
||||
// `ppc64`; only big-endian gets `endianness === 'BE'` to
|
||||
// distinguish them. Rust's two arch values both map here.
|
||||
"powerpc64" | "powerpc64le" => "ppc64",
|
||||
"loongarch64" => "loong64",
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::engine_name;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
/// Format matches pnpm's `${platform};${arch};node${major}`
|
||||
/// — required for the side-effects cache to interop.
|
||||
#[test]
|
||||
fn engine_name_matches_pnpm_format() {
|
||||
assert_eq!(engine_name(20, Some("darwin"), Some("arm64")), "darwin;arm64;node20");
|
||||
assert_eq!(engine_name(22, Some("linux"), Some("x64")), "linux;x64;node22");
|
||||
assert_eq!(engine_name(24, Some("win32"), Some("x64")), "win32;x64;node24");
|
||||
}
|
||||
|
||||
/// Defaults route through the host mapping. Just assert the
|
||||
/// shape (three semicolon-separated parts ending in
|
||||
/// `node<digits>`) — the exact OS/arch depends on where the
|
||||
/// test is run.
|
||||
#[test]
|
||||
fn engine_name_host_default_has_expected_shape() {
|
||||
let name = engine_name(20, None, None);
|
||||
let parts: Vec<&str> = name.split(';').collect();
|
||||
assert_eq!(parts.len(), 3, "expected three parts, got {name:?}");
|
||||
assert!(parts[2].starts_with("node"), "third part must start with `node`: {name:?}");
|
||||
assert!(parts[2][4..].parse::<u32>().is_ok(), "node version must be numeric: {name:?}");
|
||||
}
|
||||
}
|
||||
32
pacquet/crates/graph-hasher/src/lib.rs
Normal file
32
pacquet/crates/graph-hasher/src/lib.rs
Normal file
@@ -0,0 +1,32 @@
|
||||
//! Pacquet port of pnpm's `@pnpm/crypto.object-hasher` (which itself
|
||||
//! wraps npm's `object-hash` library) and `@pnpm/deps.graph-hasher`.
|
||||
//!
|
||||
//! The cache key used by the side-effects cache is **on disk and
|
||||
//! shared with pnpm**, so the hash output must be byte-for-byte
|
||||
//! identical to what pnpm produces. The `object-hash` library uses a
|
||||
//! specific recursive bytestream format under the hood
|
||||
//! (`object:<N>:<key>:<value>,...` with sorted keys, `string:<utf16_len>:<value>`,
|
||||
//! etc.) — pacquet replicates that format here.
|
||||
//!
|
||||
//! References (pinned to b4f8f47ac2 / object-hash@3.0.0):
|
||||
//! - <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/src/index.ts>
|
||||
//! - <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts>
|
||||
//! - <https://github.com/puleos/object-hash/blob/v3.0.0/index.js>
|
||||
|
||||
mod dep_state;
|
||||
mod engine_name;
|
||||
mod object_hasher;
|
||||
|
||||
pub use dep_state::{CalcDepStateOptions, DepsStateCache, calc_dep_state};
|
||||
pub use engine_name::engine_name;
|
||||
pub use object_hasher::{hash_object, hash_object_with_encoding, hash_object_without_sorting};
|
||||
|
||||
/// Hex/base64 encoding option for [`hash_object_with_encoding`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HashEncoding {
|
||||
Base64,
|
||||
Hex,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
144
pacquet/crates/graph-hasher/src/object_hasher.rs
Normal file
144
pacquet/crates/graph-hasher/src/object_hasher.rs
Normal file
@@ -0,0 +1,144 @@
|
||||
use crate::HashEncoding;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Mirrors `hashObject` from
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/src/index.ts#L41>
|
||||
/// (sorted keys, sha256, base64).
|
||||
///
|
||||
/// The bytestream the library writes before hashing is described in
|
||||
/// the (private) `serialize` helper below — it must match pnpm's
|
||||
/// byte-for-byte because the result is persisted on disk and shared
|
||||
/// with pnpm.
|
||||
///
|
||||
/// `undefined` in JS maps to no Rust value here; the upstream
|
||||
/// short-circuit `hashUnknown(undefined)` returns 44 zero characters
|
||||
/// regardless of options. Callers who need that semantic should
|
||||
/// branch on the optional before calling.
|
||||
pub fn hash_object(value: &Value) -> String {
|
||||
hash_object_with_encoding(value, HashEncoding::Base64, /* sort */ true)
|
||||
}
|
||||
|
||||
/// Mirrors `hashObjectWithoutSorting` at
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/src/index.ts#L37>.
|
||||
pub fn hash_object_without_sorting(value: &Value, encoding: HashEncoding) -> String {
|
||||
hash_object_with_encoding(value, encoding, /* sort */ false)
|
||||
}
|
||||
|
||||
/// General form. `sort = true` sorts object keys before serialization
|
||||
/// (the `unorderedObjects` option upstream); `sort = false` preserves
|
||||
/// insertion order.
|
||||
pub fn hash_object_with_encoding(value: &Value, encoding: HashEncoding, sort: bool) -> String {
|
||||
let mut bytes = Vec::new();
|
||||
serialize(&mut bytes, value, sort);
|
||||
let digest = Sha256::digest(&bytes);
|
||||
match encoding {
|
||||
HashEncoding::Base64 => BASE64.encode(digest),
|
||||
HashEncoding::Hex => format!("{digest:x}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursive bytestream serializer mirroring the `typeHasher` dispatch
|
||||
/// in object-hash@3.0.0 with `respectType: false`. Only the type
|
||||
/// arms pacquet actually feeds in are implemented here — `Value` is
|
||||
/// the union of String / Number / Bool / Null / Array / Object,
|
||||
/// which covers every input pacquet's graph-hasher and the upstream
|
||||
/// `hashObject`-using callers ever pass. Anything else would either
|
||||
/// be unreachable for pacquet or require porting upstream's Date /
|
||||
/// Set / Map / Buffer arms — explicitly not in scope here.
|
||||
fn serialize(out: &mut Vec<u8>, value: &Value, sort: bool) {
|
||||
match value {
|
||||
Value::Null => {
|
||||
// object-hash writes the literal string `Null` (uppercase
|
||||
// `N`) for null values at index.js:334.
|
||||
out.extend_from_slice(b"Null");
|
||||
}
|
||||
Value::Bool(b) => {
|
||||
// index.js:301-303.
|
||||
out.extend_from_slice(b"bool:");
|
||||
out.extend_from_slice(if *b { b"true" } else { b"false" });
|
||||
}
|
||||
Value::Number(n) => {
|
||||
// index.js:327-329 — `number:<n.toString()>`. JS
|
||||
// `Number.prototype.toString()` for integers gives the
|
||||
// plain decimal repr; for floats it produces the
|
||||
// shortest round-trippable form. Pacquet's cache-key
|
||||
// inputs only ever hit the integer path in practice
|
||||
// (`hash_object` is called by `calc_dep_graph_hash`
|
||||
// with strings + nested objects of strings; the
|
||||
// upstream `hashObject` test cases use integer
|
||||
// literals `1`, `2`, `3`). `n.to_string()` from
|
||||
// `serde_json::Number` matches for integers; for
|
||||
// non-integer floats `serde_json`'s `f64` formatting
|
||||
// can diverge from JS's "shortest round-trippable"
|
||||
// (ECMA-262 §7.1.12.1). If a caller ever passes a
|
||||
// non-integer in the hot path, the divergence would
|
||||
// break cross-tool cache-key parity — add the
|
||||
// JS-compatible float formatter (e.g. the `ryu` crate
|
||||
// gated on a check) before that lands.
|
||||
out.extend_from_slice(b"number:");
|
||||
out.extend_from_slice(n.to_string().as_bytes());
|
||||
}
|
||||
Value::String(s) => serialize_str(out, s),
|
||||
Value::Array(arr) => {
|
||||
// index.js:257-291 — `array:<N>:` then each entry.
|
||||
// pnpm's `hashObject` *does* sort arrays in the
|
||||
// unordered case, but pacquet does not currently feed
|
||||
// arrays through this path, so the simpler "ordered"
|
||||
// variant is what we model. Adding the unordered
|
||||
// permutation handling can land alongside a real
|
||||
// caller that needs it.
|
||||
out.extend_from_slice(b"array:");
|
||||
out.extend_from_slice(arr.len().to_string().as_bytes());
|
||||
out.push(b':');
|
||||
for entry in arr {
|
||||
serialize(out, entry, sort);
|
||||
}
|
||||
}
|
||||
Value::Object(map) => {
|
||||
// index.js:225-255 — `object:<N>:<key>:<value>,...`.
|
||||
// Sorted iff `unorderedObjects` (i.e. `sort = true`).
|
||||
// Each key is dispatched through `_string` and each
|
||||
// value through the normal dispatcher.
|
||||
out.extend_from_slice(b"object:");
|
||||
out.extend_from_slice(map.len().to_string().as_bytes());
|
||||
out.push(b':');
|
||||
if sort {
|
||||
let mut keys: Vec<&String> = map.keys().collect();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
write_pair(out, key, &map[key], sort);
|
||||
}
|
||||
} else {
|
||||
for (key, val) in map {
|
||||
write_pair(out, key, val, sort);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_pair(out: &mut Vec<u8>, key: &str, value: &Value, sort: bool) {
|
||||
// `serialize` for a `Value::String` would force a per-key
|
||||
// `Value::String(key.to_string())` allocation. Inline the
|
||||
// string framing instead so hashing a 1000-key object doesn't
|
||||
// allocate 1000 short-lived Strings on the hot path.
|
||||
serialize_str(out, key);
|
||||
out.push(b':');
|
||||
serialize(out, value, sort);
|
||||
out.push(b',');
|
||||
}
|
||||
|
||||
/// Emit the `string:<utf16_len>:<value>` framing from object-hash's
|
||||
/// `_string` arm (index.js:304-307). `length` is UTF-16 code units
|
||||
/// (JS `.length`), not bytes and not Unicode codepoints. For ASCII
|
||||
/// strings all three agree.
|
||||
fn serialize_str(out: &mut Vec<u8>, s: &str) {
|
||||
let utf16_len: usize = s.encode_utf16().count();
|
||||
out.extend_from_slice(b"string:");
|
||||
out.extend_from_slice(utf16_len.to_string().as_bytes());
|
||||
out.push(b':');
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
120
pacquet/crates/graph-hasher/src/tests.rs
Normal file
120
pacquet/crates/graph-hasher/src/tests.rs
Normal file
@@ -0,0 +1,120 @@
|
||||
use crate::{HashEncoding, hash_object, hash_object_with_encoding, hash_object_without_sorting};
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
/// Ports `hashObject` `creates a hash` from
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L6>.
|
||||
/// Pinning the exact base64 output keeps pacquet's hash byte-for-byte
|
||||
/// compatible with pnpm's `@pnpm/crypto.object-hasher` — required
|
||||
/// because the side-effects cache key is shared on disk between the
|
||||
/// two implementations.
|
||||
#[test]
|
||||
fn hash_object_known_base64_value() {
|
||||
assert_eq!(
|
||||
hash_object(&json!({ "b": 1, "a": 2 })),
|
||||
"48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ=",
|
||||
);
|
||||
}
|
||||
|
||||
/// Ports `hashObject` `sorts` from
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L10-L12>.
|
||||
/// Two objects with the same keys in a different declared order
|
||||
/// must produce the same hash.
|
||||
#[test]
|
||||
fn hash_object_sorts_object_keys() {
|
||||
assert_eq!(hash_object(&json!({ "b": 1, "a": 2 })), hash_object(&json!({ "a": 2, "b": 1 })));
|
||||
}
|
||||
|
||||
/// Ports `hashObjectWithoutSorting` `creates a hash` from
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L18>.
|
||||
/// Different exact value from the sorted variant — different key
|
||||
/// order produces different bytes.
|
||||
#[test]
|
||||
fn hash_object_without_sorting_known_base64_value() {
|
||||
assert_eq!(
|
||||
hash_object_without_sorting(&json!({ "b": 1, "a": 2 }), HashEncoding::Base64),
|
||||
"mh+rYklpd1DBj/dg6dnG+yd8BQhU2UiUoRMSXjPV1JA=",
|
||||
);
|
||||
}
|
||||
|
||||
/// Ports `hashObjectWithoutSorting` `does not sort` from
|
||||
/// <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L22-L24>.
|
||||
///
|
||||
/// `serde_json::json!` macro preserves insertion order on its
|
||||
/// `Map`-backed objects, so `{b:1,a:2}` and `{a:2,b:1}` are distinct
|
||||
/// inputs here.
|
||||
#[test]
|
||||
fn hash_object_without_sorting_distinguishes_key_order() {
|
||||
let bx = hash_object_without_sorting(&json!({ "b": 1, "a": 2 }), HashEncoding::Base64);
|
||||
let ax = hash_object_without_sorting(&json!({ "a": 2, "b": 1 }), HashEncoding::Base64);
|
||||
assert_ne!(bx, ax);
|
||||
}
|
||||
|
||||
/// Hex encoding is what `calcGraphNodeHash` uses for the GVS path
|
||||
/// at <https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L145>.
|
||||
/// Spot-check that the same input under the same options yields
|
||||
/// the hex repr of the base64 digest.
|
||||
#[test]
|
||||
fn hash_object_with_encoding_hex_matches_decoded_base64() {
|
||||
let value = json!({ "b": 1, "a": 2 });
|
||||
let base64 = hash_object(&value);
|
||||
let hex = hash_object_with_encoding(&value, HashEncoding::Hex, /* sort */ true);
|
||||
// base64 → bytes; hex → bytes; must match.
|
||||
let from_b64 =
|
||||
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, base64.as_bytes())
|
||||
.expect("decode base64");
|
||||
let from_hex = hex_decode(&hex);
|
||||
assert_eq!(from_b64, from_hex);
|
||||
}
|
||||
|
||||
/// The empty object hashes to a stable, non-empty value (the
|
||||
/// bytestream is literally `object:0:`).
|
||||
#[test]
|
||||
fn hash_object_empty_object_is_stable() {
|
||||
let h1 = hash_object(&json!({}));
|
||||
let h2 = hash_object(&json!({}));
|
||||
assert_eq!(h1, h2);
|
||||
assert!(!h1.is_empty());
|
||||
}
|
||||
|
||||
/// Nested-object case that matches the shape pacquet's
|
||||
/// `calcDepGraphHash` actually feeds into `hash_object`:
|
||||
/// `{ id: <string>, deps: <Record<string, string>> }`. Two
|
||||
/// equivalent inputs (deps in different declared order) must
|
||||
/// hash identically.
|
||||
#[test]
|
||||
fn hash_object_dep_state_shape_sorts_nested_keys() {
|
||||
let a = hash_object(&json!({
|
||||
"id": "foo@1.0.0:sha512-AAA",
|
||||
"deps": { "b": "h-b", "a": "h-a" },
|
||||
}));
|
||||
let b = hash_object(&json!({
|
||||
"deps": { "a": "h-a", "b": "h-b" },
|
||||
"id": "foo@1.0.0:sha512-AAA",
|
||||
}));
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
/// Non-ASCII string lengths must be counted in UTF-16 code units
|
||||
/// (JS `string.length`), not in bytes or codepoints. Verifies the
|
||||
/// `string:<utf16_len>:<value>` framing.
|
||||
#[test]
|
||||
fn hash_object_string_length_counts_utf16_code_units() {
|
||||
// `"é"` is one codepoint, two UTF-8 bytes, one UTF-16 code
|
||||
// unit. `"😀"` is one codepoint, four UTF-8 bytes, two UTF-16
|
||||
// code units (surrogate pair).
|
||||
let one_unit = hash_object(&json!({ "k": "é" }));
|
||||
// Construct an object whose string would be the same length
|
||||
// as `"é"` under a wrong byte-length implementation, so the
|
||||
// hashes would collide if pacquet measured bytes.
|
||||
let two_bytes_ascii = hash_object(&json!({ "k": "ab" }));
|
||||
assert_ne!(one_unit, two_bytes_ascii, "utf-16 length must differ from byte length");
|
||||
}
|
||||
|
||||
fn hex_decode(hex: &str) -> Vec<u8> {
|
||||
assert!(hex.len() % 2 == 0);
|
||||
(0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("hex digit"))
|
||||
.collect()
|
||||
}
|
||||
@@ -36,6 +36,7 @@ fn create_config(store_dir: &Path, modules_dir: &Path, virtual_store_dir: &Path)
|
||||
strict_peer_dependencies: false,
|
||||
resolve_peers_from_workspace_root: false,
|
||||
verify_store_integrity: true,
|
||||
side_effects_cache: true,
|
||||
fetch_retries: 2,
|
||||
fetch_retry_factor: 10,
|
||||
fetch_retry_mintimeout: 10_000,
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
//! cross-reference (or a pnpm-side change we need to match) stays
|
||||
//! cheap.
|
||||
|
||||
use crate::{CafsFileInfo, PackageFilesIndex, StoreDir};
|
||||
use crate::{CafsFileInfo, PackageFilesIndex, SideEffectsDiff, StoreDir};
|
||||
use dashmap::DashSet;
|
||||
use sha2::{Digest, Sha512};
|
||||
use std::{
|
||||
@@ -60,10 +60,22 @@ pub type FilesMap = HashMap<String, PathBuf>;
|
||||
/// path` map; it may be partial or empty when a digest in the index
|
||||
/// row couldn't be reconstructed into a CAFS path, so callers should
|
||||
/// gate reuse on `passed` rather than on the map's size.
|
||||
///
|
||||
/// `side_effects_maps` is the optional cache-key → overlaid-FilesMap
|
||||
/// table from a populated side-effects cache (typically seeded by
|
||||
/// pnpm). Each value is the post-build files map for one cache key:
|
||||
/// the base `files_map` with the entry's `added` overlay applied on
|
||||
/// top of it and `deleted` entries dropped. Mirrors
|
||||
/// [`PackageFilesResponse.sideEffectsMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L83-L100)
|
||||
/// — the importer looks up the entry by the dep-state cache key
|
||||
/// (`<engine>` or `<engine>;deps=…;patch=…`, produced by
|
||||
/// `pacquet-graph-hasher`'s `calc_dep_state`) to decide whether
|
||||
/// the package is already built.
|
||||
#[derive(Debug)]
|
||||
pub struct VerifyResult {
|
||||
pub passed: bool,
|
||||
pub files_map: FilesMap,
|
||||
pub side_effects_maps: Option<HashMap<String, FilesMap>>,
|
||||
}
|
||||
|
||||
/// Fast path used when `verify-store-integrity` is `false`.
|
||||
@@ -74,13 +86,14 @@ pub struct VerifyResult {
|
||||
/// corrupt CAFS file surfaces lazily at import time (pnpm's `linkOrCopy`
|
||||
/// equivalent).
|
||||
pub fn build_file_maps_from_index(store_dir: &StoreDir, entry: PackageFilesIndex) -> VerifyResult {
|
||||
let mut files_map = HashMap::with_capacity(entry.files.len());
|
||||
let PackageFilesIndex { files, side_effects, .. } = entry;
|
||||
let mut files_map = HashMap::with_capacity(files.len());
|
||||
let mut passed = true;
|
||||
// Consume `entry.files` so the owned `String` filenames move into
|
||||
// `files_map` without a per-file clone. On a realistic install the
|
||||
// previous borrow-then-clone cost one allocation per file on every
|
||||
// warm cache hit.
|
||||
for (filename, info) in entry.files {
|
||||
for (filename, info) in files {
|
||||
let Some(path) = store_dir.cas_file_path_by_mode(&info.digest, info.mode) else {
|
||||
// A malformed digest (non-hex / too short) makes this entry
|
||||
// unreconstructable. pnpm's `getFilePathByModeInCafs` doesn't
|
||||
@@ -103,7 +116,8 @@ pub fn build_file_maps_from_index(store_dir: &StoreDir, entry: PackageFilesIndex
|
||||
};
|
||||
files_map.insert(filename, path);
|
||||
}
|
||||
VerifyResult { passed, files_map }
|
||||
let side_effects_maps = build_side_effects_maps(store_dir, side_effects, &files_map);
|
||||
VerifyResult { passed, files_map, side_effects_maps }
|
||||
}
|
||||
|
||||
/// Careful path used when `verify-store-integrity` is `true` (pnpm's
|
||||
@@ -134,7 +148,7 @@ pub fn check_pkg_files_integrity(
|
||||
// Destructure so the owned `files` HashMap and `algo` String can be
|
||||
// consumed below; moving beats the extra per-file `filename.clone()`
|
||||
// the old borrow-based signature forced on the hot path.
|
||||
let PackageFilesIndex { files, algo, .. } = entry;
|
||||
let PackageFilesIndex { files, algo, side_effects, .. } = entry;
|
||||
let mut all_verified = true;
|
||||
let mut files_map = HashMap::with_capacity(files.len());
|
||||
// `verified_files_cache` is the install-scoped
|
||||
@@ -180,7 +194,71 @@ pub fn check_pkg_files_integrity(
|
||||
}
|
||||
files_map.insert(filename, path);
|
||||
}
|
||||
VerifyResult { passed: all_verified, files_map }
|
||||
let side_effects_maps = build_side_effects_maps(store_dir, side_effects, &files_map);
|
||||
VerifyResult { passed: all_verified, files_map, side_effects_maps }
|
||||
}
|
||||
|
||||
/// Materialize the per-cache-key overlaid `FilesMap`s from a
|
||||
/// `PackageFilesIndex.side_effects` entry. Mirrors upstream's
|
||||
/// [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121):
|
||||
/// the overlay is `added` plus base entries that aren't in `deleted`,
|
||||
/// with `added` winning when both name the same filename. The
|
||||
/// content of `added` entries is *not* re-verified here — pnpm
|
||||
/// doesn't do that either; corruption in the side-effects layer
|
||||
/// would surface at import time via `linkOrCopy` failing on a
|
||||
/// missing CAS blob.
|
||||
///
|
||||
/// Returns `None` when the entry has no `side_effects` field (the
|
||||
/// common case for pacquet-written rows today) so callers can
|
||||
/// trivially distinguish "no cache configured" from "cache configured
|
||||
/// but empty".
|
||||
fn build_side_effects_maps(
|
||||
store_dir: &StoreDir,
|
||||
side_effects: Option<HashMap<String, SideEffectsDiff>>,
|
||||
base_files: &FilesMap,
|
||||
) -> Option<HashMap<String, FilesMap>> {
|
||||
let raw = side_effects?;
|
||||
let mut out: HashMap<String, FilesMap> = HashMap::with_capacity(raw.len());
|
||||
'next_key: for (cache_key, diff) in raw {
|
||||
let SideEffectsDiff { added, deleted } = diff;
|
||||
let mut overlay: FilesMap = HashMap::with_capacity(base_files.len());
|
||||
if let Some(added) = added {
|
||||
for (filename, info) in added {
|
||||
let Some(path) = store_dir.cas_file_path_by_mode(&info.digest, info.mode) else {
|
||||
// Skip the entire `cache_key` entry rather than
|
||||
// returning a partial overlay. A future importer
|
||||
// that flips `is_built = true` on overlay
|
||||
// presence would otherwise turn a malformed
|
||||
// digest into a silent corruption: build skipped
|
||||
// but a required artifact missing from disk.
|
||||
// Dropping the whole entry sends the package back
|
||||
// through the rebuild path, which is safe.
|
||||
tracing::debug!(
|
||||
target: "pacquet::store_index",
|
||||
?filename,
|
||||
digest = %info.digest,
|
||||
cache_key,
|
||||
"malformed CAFS digest in side-effects `added` overlay; dropping this cache_key entry entirely so the importer falls back to rebuild",
|
||||
);
|
||||
continue 'next_key;
|
||||
};
|
||||
overlay.insert(filename, path);
|
||||
}
|
||||
}
|
||||
// Promote `deleted` to a `HashSet` once per cache key so
|
||||
// the `base_files` walk stays linear in `|base|` instead of
|
||||
// `O(|base| * |deleted|)`. Pnpm's TS side keeps `deleted`
|
||||
// as a `Set` for the same reason.
|
||||
let deleted_set: std::collections::HashSet<String> =
|
||||
deleted.unwrap_or_default().into_iter().collect();
|
||||
for (filename, path) in base_files {
|
||||
if !deleted_set.contains(filename) && !overlay.contains_key(filename) {
|
||||
overlay.insert(filename.clone(), path.clone());
|
||||
}
|
||||
}
|
||||
out.insert(cache_key, overlay);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Port of pnpm's `verifyFile`. `true` when the on-disk file is either
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use super::{VerifiedFilesCache, build_file_maps_from_index, check_pkg_files_integrity};
|
||||
use crate::{CafsFileInfo, PackageFilesIndex, StoreDir};
|
||||
use crate::{CafsFileInfo, PackageFilesIndex, SideEffectsDiff, StoreDir};
|
||||
use pretty_assertions::assert_eq;
|
||||
use sha2::{Digest, Sha512};
|
||||
use std::collections::HashMap;
|
||||
use std::{
|
||||
fs,
|
||||
io::Write,
|
||||
@@ -321,3 +322,175 @@ impl CafsFileInfo {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// No `side_effects` field on the index → `VerifyResult.side_effects_maps`
|
||||
/// is `None`. Distinguishes "this package never had a cache entry
|
||||
/// written" from "cache configured but empty for this key" — the
|
||||
/// importer treats the former as a regular non-built import.
|
||||
#[test]
|
||||
fn no_side_effects_yields_none() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let store_dir = StoreDir::new(tmp.path());
|
||||
let digest = sha512_hex(b"x");
|
||||
let entry = index_with("sha512", vec![("a", info(&digest, 1, 0o644, None))]);
|
||||
let result = build_file_maps_from_index(&store_dir, entry);
|
||||
assert!(result.side_effects_maps.is_none());
|
||||
}
|
||||
|
||||
/// One cache key, one `added` file, one `deleted` file: the
|
||||
/// overlay is `added` ∪ (base \ deleted) — entries in `added` win
|
||||
/// when both layers name the same filename. Mirrors upstream's
|
||||
/// [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121).
|
||||
#[test]
|
||||
fn side_effects_overlay_adds_and_drops_correctly() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let store_dir = StoreDir::new(tmp.path());
|
||||
let base_digest = sha512_hex(b"base");
|
||||
let added_digest = sha512_hex(b"added");
|
||||
// base files: a.js, b.js. Side-effects for one cache key:
|
||||
// add c.js, delete b.js. Overlay should land {a.js, c.js}.
|
||||
let mut side_effects = HashMap::new();
|
||||
let mut added = HashMap::new();
|
||||
added.insert("c.js".to_string(), info(&added_digest, 5, 0o644, None));
|
||||
side_effects.insert(
|
||||
"darwin;arm64;node20;deps=fake".to_string(),
|
||||
SideEffectsDiff { added: Some(added), deleted: Some(vec!["b.js".to_string()]) },
|
||||
);
|
||||
let entry = PackageFilesIndex {
|
||||
manifest: None,
|
||||
requires_build: None,
|
||||
algo: "sha512".into(),
|
||||
files: HashMap::from([
|
||||
("a.js".to_string(), info(&base_digest, 4, 0o644, None)),
|
||||
("b.js".to_string(), info(&base_digest, 4, 0o644, None)),
|
||||
]),
|
||||
side_effects: Some(side_effects),
|
||||
};
|
||||
let result = build_file_maps_from_index(&store_dir, entry);
|
||||
let maps = result.side_effects_maps.expect("populated");
|
||||
let overlay = maps.get("darwin;arm64;node20;deps=fake").expect("entry exists");
|
||||
assert!(overlay.contains_key("a.js"), "base survives: {overlay:?}");
|
||||
assert!(overlay.contains_key("c.js"), "added overlays: {overlay:?}");
|
||||
assert!(!overlay.contains_key("b.js"), "deleted drops: {overlay:?}");
|
||||
assert_eq!(overlay.len(), 2);
|
||||
}
|
||||
|
||||
/// `added` wins over `base` when the filenames collide. The base
|
||||
/// path is shadowed by the side-effects path.
|
||||
#[test]
|
||||
fn side_effects_overlay_added_shadows_base_on_collision() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let store_dir = StoreDir::new(tmp.path());
|
||||
let base_digest = sha512_hex(b"base");
|
||||
let overlay_digest = sha512_hex(b"overlay-shadow");
|
||||
let mut added = HashMap::new();
|
||||
added.insert("collide.js".to_string(), info(&overlay_digest, 16, 0o644, None));
|
||||
let mut side_effects = HashMap::new();
|
||||
side_effects.insert("k1".to_string(), SideEffectsDiff { added: Some(added), deleted: None });
|
||||
let entry = PackageFilesIndex {
|
||||
manifest: None,
|
||||
requires_build: None,
|
||||
algo: "sha512".into(),
|
||||
files: HashMap::from([("collide.js".to_string(), info(&base_digest, 4, 0o644, None))]),
|
||||
side_effects: Some(side_effects),
|
||||
};
|
||||
let result = build_file_maps_from_index(&store_dir, entry);
|
||||
let overlay = result.side_effects_maps.unwrap().remove("k1").unwrap();
|
||||
let path = overlay.get("collide.js").expect("collide.js present");
|
||||
// CAFS layout splits the digest as `<2-char prefix>/<rest>`, so the
|
||||
// path won't contain the digest as a single contiguous substring.
|
||||
// Verify by checking that the overlay digest's tail (post-prefix
|
||||
// hex) appears in the path, and that the base digest's tail does
|
||||
// NOT.
|
||||
let path_str = path.to_string_lossy();
|
||||
assert!(
|
||||
path_str.contains(&overlay_digest[2..]),
|
||||
"overlay digest tail should appear in CAFS path: {path:?}",
|
||||
);
|
||||
assert!(
|
||||
!path_str.contains(&base_digest[2..]),
|
||||
"base digest tail must NOT appear (shadowed): {path:?}",
|
||||
);
|
||||
}
|
||||
|
||||
/// A malformed digest inside an `added` overlay drops the **whole**
|
||||
/// cache_key entry — not just the single bad file. Mismatched
|
||||
/// overlays would otherwise turn a future `is_built = true` decision
|
||||
/// into a silent corruption (build skipped, required artifact
|
||||
/// missing). Other cache_key entries on the same package survive.
|
||||
#[test]
|
||||
fn side_effects_overlay_malformed_added_digest_drops_cache_key_entry() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let store_dir = StoreDir::new(tmp.path());
|
||||
let base_digest = sha512_hex(b"base");
|
||||
let good_digest = sha512_hex(b"good-added");
|
||||
|
||||
let mut k_bad_added = HashMap::new();
|
||||
// One good file alongside one bad one — the whole entry should
|
||||
// still drop, not just the bad file.
|
||||
k_bad_added.insert("good.js".to_string(), info(&good_digest, 4, 0o644, None));
|
||||
k_bad_added.insert("bad.js".to_string(), info("not-hex", 4, 0o644, None));
|
||||
|
||||
let mut k_good_added = HashMap::new();
|
||||
k_good_added.insert("ok.js".to_string(), info(&good_digest, 4, 0o644, None));
|
||||
|
||||
let mut side_effects = HashMap::new();
|
||||
side_effects
|
||||
.insert("k-bad".to_string(), SideEffectsDiff { added: Some(k_bad_added), deleted: None });
|
||||
side_effects
|
||||
.insert("k-good".to_string(), SideEffectsDiff { added: Some(k_good_added), deleted: None });
|
||||
|
||||
let entry = PackageFilesIndex {
|
||||
manifest: None,
|
||||
requires_build: None,
|
||||
algo: "sha512".into(),
|
||||
files: HashMap::from([("base.js".to_string(), info(&base_digest, 4, 0o644, None))]),
|
||||
side_effects: Some(side_effects),
|
||||
};
|
||||
let result = build_file_maps_from_index(&store_dir, entry);
|
||||
let maps = result.side_effects_maps.expect("populated");
|
||||
assert!(!maps.contains_key("k-bad"), "k-bad must drop entirely on malformed digest");
|
||||
assert!(maps.contains_key("k-good"), "k-good must survive: {maps:?}");
|
||||
}
|
||||
|
||||
/// Multiple cache keys produce independent overlays. One entry's
|
||||
/// `added` doesn't bleed into another's.
|
||||
#[test]
|
||||
fn side_effects_overlay_keys_are_independent() {
|
||||
let tmp = tempdir().unwrap();
|
||||
let store_dir = StoreDir::new(tmp.path());
|
||||
let base_digest = sha512_hex(b"base");
|
||||
let added_k1 = sha512_hex(b"k1-added");
|
||||
let added_k2 = sha512_hex(b"k2-added");
|
||||
let mut side_effects = HashMap::new();
|
||||
side_effects.insert(
|
||||
"k1".to_string(),
|
||||
SideEffectsDiff {
|
||||
added: Some(HashMap::from([("a.js".to_string(), info(&added_k1, 1, 0o644, None))])),
|
||||
deleted: None,
|
||||
},
|
||||
);
|
||||
side_effects.insert(
|
||||
"k2".to_string(),
|
||||
SideEffectsDiff {
|
||||
added: Some(HashMap::from([("b.js".to_string(), info(&added_k2, 1, 0o644, None))])),
|
||||
deleted: None,
|
||||
},
|
||||
);
|
||||
let entry = PackageFilesIndex {
|
||||
manifest: None,
|
||||
requires_build: None,
|
||||
algo: "sha512".into(),
|
||||
files: HashMap::from([("base.js".to_string(), info(&base_digest, 4, 0o644, None))]),
|
||||
side_effects: Some(side_effects),
|
||||
};
|
||||
let result = build_file_maps_from_index(&store_dir, entry);
|
||||
let maps = result.side_effects_maps.unwrap();
|
||||
let k1 = maps.get("k1").unwrap();
|
||||
let k2 = maps.get("k2").unwrap();
|
||||
assert!(k1.contains_key("a.js") && !k1.contains_key("b.js"), "k1: {k1:?}");
|
||||
assert!(k2.contains_key("b.js") && !k2.contains_key("a.js"), "k2: {k2:?}");
|
||||
// Both share base.js.
|
||||
assert!(k1.contains_key("base.js"));
|
||||
assert!(k2.contains_key("base.js"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user