diff --git a/pacquet/crates/cli/tests/lockfile_resolution_reuse.rs b/pacquet/crates/cli/tests/lockfile_resolution_reuse.rs new file mode 100644 index 0000000000..f5ccf32b89 --- /dev/null +++ b/pacquet/crates/cli/tests/lockfile_resolution_reuse.rs @@ -0,0 +1,169 @@ +//! A second non-frozen install reuses the prior lockfile's resolution +//! and transitive subtree for an unchanged dependency, instead of +//! re-resolving it from the registry. +//! +//! See `pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md`. pnpm avoids +//! re-resolving an unchanged tree by reading the prior lockfile's +//! recorded resolution + child refs +//! ([`getInfoFromLockfile`](https://github.com/pnpm/pnpm/blob/097983fbca/installing/deps-resolver/src/resolveDependencies.ts#L1199-L1248)); +//! pacquet ports that so a re-install with the registry gone still +//! succeeds for the unchanged subtree. +//! +//! The proof, modeled on `tarball_url_dependency.rs`'s +//! `remote_tarball_reresolves_from_warm_store_without_refetch`: a fresh +//! install against the live mock registry warms the store and records +//! the lockfile (a direct dep plus its one transitive dep); the registry +//! is then repointed at a dead port; finally a non-frozen install — which +//! goes through the fresh-lockfile resolution path because the manifest +//! changed — must succeed. It can only succeed by reusing the unchanged +//! subtree from the lockfile, because re-resolving either package would +//! hit the dead registry and fail. + +use assert_cmd::prelude::*; +use command_extra::CommandExtra; +use pacquet_testing_utils::bin::{AddMockedRegistry, CommandTempCwd}; +use std::{fs, net::TcpListener, path::Path, process::Command}; + +fn pacquet_at(workspace: &Path) -> Command { + Command::cargo_bin("pacquet").expect("find the pacquet binary").with_current_dir(workspace) +} + +/// A `registry=` URL on a localhost port with nothing listening, so any +/// resolution attempt against it fails fast with a connection refusal. +fn dead_registry_url() -> String { + // Bind to an ephemeral port, read it, then drop the listener so the + // port is (almost certainly) free again — anything that connects to + // it gets refused. + let listener = + TcpListener::bind(("127.0.0.1", 0)).expect("bind an ephemeral port to learn a free one"); + let addr = listener.local_addr().expect("read the ephemeral port"); + drop(listener); + format!("http://127.0.0.1:{}/", addr.port()) +} + +#[test] +fn reuses_unchanged_subtree_without_re_resolving_from_the_registry() { + let CommandTempCwd { workspace, root, npmrc_info, .. } = + CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { mock_instance, npmrc_path, .. } = npmrc_info; + + // Disable `minimumReleaseAge` so the post-resolution lockfile + // verifier doesn't fetch each entry's metadata from the registry — + // that fetch is a separate concern from resolution reuse, and with + // the default (1 day) it would hit the dead registry regardless of + // whether resolution was reused, masking what this test proves. + let workspace_yaml = workspace.join("pnpm-workspace.yaml"); + let existing = fs::read_to_string(&workspace_yaml).expect("read pnpm-workspace.yaml"); + fs::write(&workspace_yaml, format!("{existing}minimumReleaseAge: 0\n")) + .expect("append minimumReleaseAge to pnpm-workspace.yaml"); + + // `@pnpm.e2e/pkg-with-1-dep@100.0.0` depends on + // `@pnpm.e2e/dep-of-pkg-with-1-dep@^100.0.0`, so the lockfile records + // a two-node subtree (the direct dep plus its transitive child). + let manifest_path = workspace.join("package.json"); + let lockfile_path = workspace.join("pnpm-lock.yaml"); + fs::write( + &manifest_path, + serde_json::json!({ "dependencies": { "@pnpm.e2e/pkg-with-1-dep": "100.0.0" } }) + .to_string(), + ) + .expect("write package.json"); + + // Fresh install against the live registry: warms the store and writes + // the lockfile. + pacquet_at(&workspace).with_arg("install").assert().success(); + + let lockfile = fs::read_to_string(&lockfile_path).expect("read pnpm-lock.yaml"); + assert!( + lockfile.contains("@pnpm.e2e/pkg-with-1-dep@100.0.0") + && lockfile.contains("@pnpm.e2e/dep-of-pkg-with-1-dep@"), + "the fresh install must record the direct dep and its transitive child:\n{lockfile}", + ); + + // Repoint the registry at a dead port. Any re-resolution now fails. + let dead_registry = dead_registry_url(); + let npmrc = fs::read_to_string(&npmrc_path).expect("read .npmrc"); + let npmrc = npmrc + .lines() + .filter(|line| !line.trim_start().starts_with("registry=")) + .collect::>() + .join("\n"); + fs::write(&npmrc_path, format!("registry={dead_registry}\n{npmrc}\n")) + .expect("rewrite .npmrc with a dead registry"); + + // Widen the range to `^100.0.0`. The locked `100.0.0` still satisfies + // it (so the dep is reusable), but the manifest change forces the + // non-frozen fresh-lockfile resolution path rather than the + // up-to-date short-circuit. + fs::write( + &manifest_path, + serde_json::json!({ "dependencies": { "@pnpm.e2e/pkg-with-1-dep": "^100.0.0" } }) + .to_string(), + ) + .expect("rewrite package.json with a widened range"); + + // Succeeds only because the unchanged subtree is reused from the + // lockfile — re-resolving either package would hit the dead registry. + pacquet_at(&workspace).with_arg("install").assert().success(); + + drop((root, mock_instance)); +} + +/// A lockfile produced via the reuse path is structurally identical to +/// one produced by resolving the same manifest entirely from scratch. +/// +/// The discriminating test above proves reuse *fires*; this proves it's +/// *correct* — that reusing an unchanged subtree yields the same tree a +/// fresh resolve would, so reuse can never silently drift the resolution. +/// Reaching the same final manifest two ways: +/// A. install `pkg-with-1-dep`, then add `foo` — the second install +/// reuses `pkg-with-1-dep`'s subtree and resolves only `foo`; +/// B. install both from scratch — no prior lockfile, nothing reused. +/// +/// Compared as parsed [`pacquet_lockfile::Lockfile`] values rather than +/// raw bytes: the two are content-identical (same packages, versions, +/// integrities, snapshots, importer specifiers), but the writer emits the +/// `packages` / `snapshots` / importer-`dependencies` maps in build- +/// insertion order, which differs between the incremental and the fresh +/// build. That byte-level ordering is a separate lockfile-determinism +/// concern (tracked as a follow-up), orthogonal to reuse correctness. +#[test] +fn a_reused_tree_is_structurally_identical_to_a_fresh_resolve() { + let both = serde_json::json!({ + "dependencies": { "@pnpm.e2e/pkg-with-1-dep": "100.0.0", "@pnpm.e2e/foo": "100.0.0" } + }) + .to_string(); + + // Scenario A: reuse path. + let reused = CommandTempCwd::init().add_mocked_registry(); + let reused_manifest = reused.workspace.join("package.json"); + fs::write( + &reused_manifest, + serde_json::json!({ "dependencies": { "@pnpm.e2e/pkg-with-1-dep": "100.0.0" } }) + .to_string(), + ) + .expect("write the reuse scenario's initial manifest"); + pacquet_at(&reused.workspace).with_arg("install").assert().success(); + fs::write(&reused_manifest, &both).expect("add the second dep to the reuse scenario"); + pacquet_at(&reused.workspace).with_arg("install").assert().success(); + let reused_lockfile = + fs::read_to_string(reused.workspace.join("pnpm-lock.yaml")).expect("read reused lockfile"); + + // Scenario B: fresh resolve of the same final manifest. + let fresh = CommandTempCwd::init().add_mocked_registry(); + fs::write(fresh.workspace.join("package.json"), &both).expect("write the fresh manifest"); + pacquet_at(&fresh.workspace).with_arg("install").assert().success(); + let fresh_lockfile = + fs::read_to_string(fresh.workspace.join("pnpm-lock.yaml")).expect("read fresh lockfile"); + + let parse = |yaml: &str| { + serde_saphyr::from_str::(yaml).expect("parse pnpm-lock.yaml") + }; + pretty_assertions::assert_eq!( + parse(&reused_lockfile), + parse(&fresh_lockfile), + "a tree built via subtree reuse must be structurally identical to a fresh resolve", + ); + + drop((reused, fresh)); +} diff --git a/pacquet/crates/package-manager/src/install_with_fresh_lockfile.rs b/pacquet/crates/package-manager/src/install_with_fresh_lockfile.rs index 5a456ebf3a..ba5a631b6f 100644 --- a/pacquet/crates/package-manager/src/install_with_fresh_lockfile.rs +++ b/pacquet/crates/package-manager/src/install_with_fresh_lockfile.rs @@ -824,6 +824,34 @@ impl<'a, DependencyGroupList> InstallWithFreshLockfile<'a, DependencyGroupList> manifest_hook: package_extensions_hook.clone(), pick_lowest_direct, time_based, + // Hand the resolver the prior lockfile so it can reuse + // already-resolved subtrees instead of re-resolving from the + // registry (see pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md). + // Withhold it when `packageExtensions` drifted: a changed + // extension rewrites packages' dependency sets, so the recorded + // subtree is stale — pnpm likewise invalidates the lockfile on a + // settings change. (`overrides` are applied to the manifest + // before the importer-level reuse gate re-checks the specifier; + // a follow-up should also guard transitive reuse against + // overrides drift.) + wanted_lockfile: wanted_lockfile + .filter(|lockfile| { + lockfile.package_extensions_checksum + == compute_package_extensions_checksum(config) + }) + .cloned() + .map(Arc::new), + // `pacquet update` must re-resolve its targets to highest- + // in-range, so suppress reuse for them (and their subtrees). + update_reuse_scope: match &update_seed_policy { + UpdateSeedPolicy::KeepAll => pacquet_resolving_deps_resolver::UpdateReuseScope::All, + UpdateSeedPolicy::DropAll => { + pacquet_resolving_deps_resolver::UpdateReuseScope::None + } + UpdateSeedPolicy::DropOnly(names) => { + pacquet_resolving_deps_resolver::UpdateReuseScope::Except(names.clone()) + } + }, }; let modules_basename = config .modules_dir diff --git a/pacquet/crates/resolving-deps-resolver/src/hoist_peers.rs b/pacquet/crates/resolving-deps-resolver/src/hoist_peers.rs index 1ef05ccb01..5819d6527f 100644 --- a/pacquet/crates/resolving-deps-resolver/src/hoist_peers.rs +++ b/pacquet/crates/resolving-deps-resolver/src/hoist_peers.rs @@ -211,7 +211,7 @@ fn max_satisfying<'a>(versions: &'a [&'a str], range: &str) -> Option<&'a str> { /// semver semantics); the retry with the prerelease tag stripped /// recovers the candidates upstream accepts. Matches the /// `satisfies_with_prereleases` pattern in the `resolve_peers` module. -fn satisfies_including_prerelease(range: &Range, version: &Version) -> bool { +pub(crate) fn satisfies_including_prerelease(range: &Range, version: &Version) -> bool { if range.satisfies(version) { return true; } diff --git a/pacquet/crates/resolving-deps-resolver/src/lib.rs b/pacquet/crates/resolving-deps-resolver/src/lib.rs index bbd077e977..d260dfc4cd 100644 --- a/pacquet/crates/resolving-deps-resolver/src/lib.rs +++ b/pacquet/crates/resolving-deps-resolver/src/lib.rs @@ -67,6 +67,7 @@ mod dedupe_injected_deps; mod dependencies_graph; mod hoist_peers; +mod lockfile_reuse; mod node_id; mod resolve_dependency_tree; mod resolve_importer; @@ -86,7 +87,7 @@ pub use node_id::NodeId; pub use pacquet_deps_path::DepPath; pub use resolve_dependency_tree::{ ManifestHook, ResolveDependencyTreeError, ResolveDependencyTreeOptions, TreeCtx, - WorkspaceTreeCtx, extend_tree, resolve_dependency_tree, + UpdateReuseScope, WorkspaceTreeCtx, extend_tree, resolve_dependency_tree, }; pub use resolve_importer::{ ResolveImporterError, ResolveImporterOptions, ResolveImporterResult, resolve_importer, diff --git a/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse.rs b/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse.rs new file mode 100644 index 0000000000..6b2ef442fd --- /dev/null +++ b/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse.rs @@ -0,0 +1,191 @@ +//! Reuse gate: decide whether the prior lockfile already satisfies a +//! wanted dependency, so the tree walker can reuse its recorded +//! resolution + subtree instead of re-resolving from the registry. +//! Mirrors pnpm's `satisfiesWanted` / `getInfoFromLockfile` gate in +//! [`resolveDependencies.ts`](https://github.com/pnpm/pnpm/blob/097983fbca/installing/deps-resolver/src/resolveDependencies.ts#L1086-L1248). +//! See `pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md`. + +use std::collections::HashMap; + +use node_semver::Range; +use pacquet_lockfile::{ + Lockfile, LockfileResolution, PkgName, PkgNameVer, PkgNameVerPeer, ProjectSnapshot, + ResolvedDependencySpec, +}; +use pacquet_resolving_resolver_base::{PkgResolutionId, ResolveResult}; +use serde_json::{Map, Value}; + +use crate::hoist_peers::satisfies_including_prerelease; + +/// The snapshot key (`snapshots:` / `packages:` map key) the prior +/// lockfile resolved `alias` to in importer `importer_id`, when the +/// recorded version still satisfies the manifest's `bare_specifier` +/// (semver-satisfies, matching pnpm's `satisfiesWanted`). +/// +/// Returns `None` — so the caller resolves fresh — for a new dependency, +/// an edited range the locked version no longer satisfies, a non-semver +/// `bare_specifier`, or a `link:` recorded shape. The first cut reuses +/// only semver (registry/tarball) deps; richer shapes (`link:`/`file:`/ +/// `workspace:`/`catalog:`) fall through to a normal resolve. +pub(crate) fn reusable_importer_dep( + importers: &HashMap, + importer_id: &str, + alias: &str, + bare_specifier: &str, +) -> Option { + let name: PkgName = alias.parse().ok()?; + let spec = importer_dep(importers.get(importer_id)?, &name)?; + let version = spec.version.ver_peer()?.version_semver()?; + let range = bare_specifier.parse::().ok()?; + if !satisfies_including_prerelease(&range, version) { + return None; + } + spec.version.resolved_key(&name) +} + +/// The recorded resolution for `name` across the importer's prod / +/// optional / dev dependency maps. +fn importer_dep<'a>( + importer: &'a ProjectSnapshot, + name: &PkgName, +) -> Option<&'a ResolvedDependencySpec> { + importer + .dependencies + .as_ref() + .and_then(|deps| deps.get(name)) + .or_else(|| importer.optional_dependencies.as_ref().and_then(|deps| deps.get(name))) + .or_else(|| importer.dev_dependencies.as_ref().and_then(|deps| deps.get(name))) +} + +/// Synthesize the [`ResolveResult`] a fresh resolve of `key` would have +/// produced, reading the recorded resolution + manifest metadata out of +/// the prior lockfile instead of hitting the registry. +/// +/// Conservative by design: returns `None` (so the caller resolves +/// fresh) unless the package is a plain-semver registry package with an +/// entry in `lockfile.packages`. pacquet's npm resolver records every +/// registry pick as a [`LockfileResolution::Tarball`] carrying the +/// registry tarball URL + integrity (it never emits the bare +/// `Registry` shape — see +/// [`npm_resolver`](https://github.com/pnpm/pnpm/blob/097983fbca/resolving/npm-resolver/src/index.ts)), +/// so both `Tarball` and `Registry` are accepted here. The +/// `version_semver()` gate keeps reuse to registry packages: a remote +/// (non-registry) tarball or git dep carries a URL-shaped, non-semver +/// version slot and falls through to a fresh resolve. Git-hosted +/// tarballs (which need preparation on extraction) are rejected +/// outright. Directory / git / binary / variations resolutions also +/// fall through — reusing them would need resolver state the lockfile +/// doesn't fully capture, and a wrong reuse produces a wrong tree. +/// +/// The synthesized result reproduces the node shape a fresh resolve +/// yields: +/// +/// * `id` / `name_ver` are the peer-stripped `name@version`, the +/// `pkgIdWithPatchHash` the dedup map keys on (the peer suffix is +/// re-derived by the peer pass). +/// * `resolution` is cloned from [`pacquet_lockfile::PackageMetadata`] +/// so the recorded integrity carries forward. +/// * `manifest` is reconstructed from the metadata's +/// `peerDependencies` / `peerDependenciesMeta` / `engines` / `cpu` / +/// `os` / `libc` / `hasBin` so `extract_peer_dependencies` +/// and the leaf classifier behave identically to a fresh resolve. +/// `dependencies` are deliberately omitted — the children come from +/// the snapshot graph, not this manifest. +pub(crate) fn synthesize_reused_result( + lockfile: &Lockfile, + key: &PkgNameVerPeer, + alias: &str, +) -> Option { + let metadata_key = key.without_peer(); + let version = metadata_key.suffix.version_semver()?.clone(); + let metadata = lockfile.packages.as_ref()?.get(&metadata_key)?; + // Reuse only registry-resolved packages for now (see the doc above). + match &metadata.resolution { + LockfileResolution::Registry(_) => {} + LockfileResolution::Tarball(tarball) + if tarball.integrity.is_some() && tarball.git_hosted != Some(true) => {} + LockfileResolution::Tarball(_) + | LockfileResolution::Directory(_) + | LockfileResolution::Git(_) + | LockfileResolution::Binary(_) + | LockfileResolution::Variations(_) => return None, + } + let name_ver = PkgNameVer::new(metadata_key.name.clone(), version); + let manifest = synthesize_manifest(&name_ver, metadata); + Some(ResolveResult { + id: PkgResolutionId::from(name_ver.to_string()), + name_ver: Some(name_ver), + latest: None, + published_at: None, + manifest: Some(std::sync::Arc::new(manifest)), + resolution: metadata.resolution.clone(), + resolved_via: "npm-registry".to_string(), + normalized_bare_specifier: None, + alias: Some(alias.to_string()), + policy_violation: None, + }) +} + +/// Reconstruct the minimal manifest fragment downstream consumers read +/// off a reused [`ResolveResult`]. Carries the peer / platform metadata +/// the lockfile records; omits `dependencies` because a reused node's +/// children come from the snapshot graph, not the manifest. +fn synthesize_manifest( + name_ver: &PkgNameVer, + metadata: &pacquet_lockfile::PackageMetadata, +) -> Value { + let mut manifest = Map::new(); + manifest.insert("name".to_string(), Value::String(name_ver.name.to_string())); + manifest.insert("version".to_string(), Value::String(name_ver.suffix.to_string())); + + if let Some(peers) = metadata.peer_dependencies.as_ref() { + let map: Map = peers + .iter() + .map(|(name, range)| (name.clone(), Value::String(range.clone()))) + .collect(); + manifest.insert("peerDependencies".to_string(), Value::Object(map)); + } + if let Some(meta) = metadata.peer_dependencies_meta.as_ref() { + let map: Map = meta + .iter() + .map(|(name, entry)| { + let mut obj = Map::new(); + obj.insert("optional".to_string(), Value::Bool(entry.optional)); + (name.clone(), Value::Object(obj)) + }) + .collect(); + manifest.insert("peerDependenciesMeta".to_string(), Value::Object(map)); + } + if let Some(engines) = metadata.engines.as_ref() { + let map: Map = engines + .iter() + .map(|(name, range)| (name.clone(), Value::String(range.clone()))) + .collect(); + manifest.insert("engines".to_string(), Value::Object(map)); + } + if let Some(cpu) = metadata.cpu.as_ref() { + manifest.insert("cpu".to_string(), string_array(cpu)); + } + if let Some(os) = metadata.os.as_ref() { + manifest.insert("os".to_string(), string_array(os)); + } + if let Some(libc) = metadata.libc.as_ref() { + manifest.insert("libc".to_string(), string_array(libc)); + } + // `has_bin: Some(true)` round-trips as a truthy `bin` so the + // bundled-manifest bin linker sees a non-empty bin set; the exact + // bin paths live in the store-index bundled manifest the install + // pass reads, not here. + if metadata.has_bin == Some(true) { + manifest.insert("bin".to_string(), Value::String(name_ver.name.to_string())); + } + + Value::Object(manifest) +} + +fn string_array(items: &[String]) -> Value { + Value::Array(items.iter().map(|item| Value::String(item.clone())).collect()) +} + +#[cfg(test)] +mod tests; diff --git a/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse/tests.rs b/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse/tests.rs new file mode 100644 index 0000000000..077485e3f2 --- /dev/null +++ b/pacquet/crates/resolving-deps-resolver/src/lockfile_reuse/tests.rs @@ -0,0 +1,158 @@ +use std::collections::HashMap; + +use pacquet_lockfile::{ + ComVer, ImporterDepVersion, Lockfile, LockfileResolution, LockfileVersion, PackageMetadata, + PkgName, PkgNameVerPeer, PkgVerPeer, ProjectSnapshot, RegistryResolution, + ResolvedDependencySpec, TarballResolution, +}; + +use super::{reusable_importer_dep, synthesize_reused_result}; + +fn single_dep_importer(alias: &str, resolved: &str) -> HashMap { + let mut deps = HashMap::new(); + deps.insert( + alias.parse::().expect("parse alias"), + ResolvedDependencySpec { + specifier: resolved.to_string(), + version: ImporterDepVersion::Regular( + resolved.parse::().expect("parse version"), + ), + }, + ); + HashMap::from([( + ".".to_string(), + ProjectSnapshot { dependencies: Some(deps), ..ProjectSnapshot::default() }, + )]) +} + +fn empty_lockfile() -> Lockfile { + Lockfile { + lockfile_version: LockfileVersion::<9>::try_from(ComVer::new(9, 0)).expect("lockfile v9"), + settings: None, + overrides: None, + package_extensions_checksum: None, + ignored_optional_dependencies: None, + importers: HashMap::new(), + packages: None, + snapshots: None, + } +} + +fn registry_metadata() -> PackageMetadata { + PackageMetadata { + resolution: LockfileResolution::Registry(RegistryResolution { + integrity: "sha512-gf6ZldcfCDyNXPRiW3lQjEP1Z9rrUM/4Cn7BZbv3SdTA82zxWRP8OmLwvGR974uuENhGCFgFdN11z3n1Ofpprg==" + .parse() + .expect("parse integrity"), + }), + engines: None, + cpu: None, + os: None, + libc: None, + deprecated: None, + has_bin: None, + prepare: None, + bundled_dependencies: None, + peer_dependencies: None, + peer_dependencies_meta: None, + } +} + +#[test] +fn reuses_when_locked_version_satisfies_the_manifest_range() { + let importers = single_dep_importer("react", "18.2.0"); + let key = reusable_importer_dep(&importers, ".", "react", "^18.0.0") + .expect("locked 18.2.0 satisfies ^18.0.0"); + assert_eq!(key.to_string(), "react@18.2.0"); +} + +#[test] +fn reuses_across_a_widened_but_still_satisfied_range() { + let importers = single_dep_importer("react", "18.2.0"); + assert!(reusable_importer_dep(&importers, ".", "react", ">=17").is_some()); +} + +#[test] +fn fresh_resolves_when_range_no_longer_satisfies_locked_version() { + let importers = single_dep_importer("react", "18.2.0"); + assert!(reusable_importer_dep(&importers, ".", "react", "^19.0.0").is_none()); +} + +#[test] +fn fresh_resolves_a_new_dependency_absent_from_the_lockfile() { + let importers = single_dep_importer("react", "18.2.0"); + assert!(reusable_importer_dep(&importers, ".", "left-pad", "^1.0.0").is_none()); +} + +#[test] +fn synthesizes_a_registry_resolution_with_the_recorded_integrity() { + let key: PkgNameVerPeer = "react@18.2.0".parse().expect("parse key"); + let metadata = registry_metadata(); + let mut lockfile = empty_lockfile(); + lockfile.packages = Some(HashMap::from([(key.clone(), metadata.clone())])); + + let result = + synthesize_reused_result(&lockfile, &key, "react").expect("registry dep is reusable"); + assert_eq!(result.id.as_str(), "react@18.2.0"); + let name_ver = result.name_ver.expect("name_ver"); + assert_eq!(name_ver.name.to_string(), "react"); + assert_eq!(name_ver.suffix.to_string(), "18.2.0"); + assert_eq!(result.resolution, metadata.resolution); + assert_eq!(result.resolved_via, "npm-registry"); + assert_eq!(result.alias.as_deref(), Some("react")); + let manifest = result.manifest.expect("synthesized manifest"); + assert_eq!(manifest.get("name").and_then(serde_json::Value::as_str), Some("react")); + assert_eq!(manifest.get("version").and_then(serde_json::Value::as_str), Some("18.2.0")); +} + +#[test] +fn synthesized_manifest_carries_peer_metadata() { + let key: PkgNameVerPeer = "react-dom@18.2.0".parse().expect("parse key"); + let mut metadata = registry_metadata(); + metadata.peer_dependencies = + Some(HashMap::from([("react".to_string(), "^18.0.0".to_string())])); + let mut lockfile = empty_lockfile(); + lockfile.packages = Some(HashMap::from([(key.clone(), metadata)])); + + let result = + synthesize_reused_result(&lockfile, &key, "react-dom").expect("registry dep is reusable"); + let manifest = result.manifest.expect("synthesized manifest"); + let peers = + manifest.get("peerDependencies").and_then(serde_json::Value::as_object).expect("peers"); + assert_eq!(peers.get("react").and_then(serde_json::Value::as_str), Some("^18.0.0")); +} + +#[test] +fn does_not_reuse_non_registry_resolutions() { + let key: PkgNameVerPeer = "pkg-from-tarball@1.0.0".parse().expect("parse key"); + let mut metadata = registry_metadata(); + metadata.resolution = LockfileResolution::Tarball(TarballResolution { + tarball: "https://example.test/pkg.tgz".to_string(), + integrity: None, + git_hosted: None, + path: None, + }); + let mut lockfile = empty_lockfile(); + lockfile.packages = Some(HashMap::from([(key.clone(), metadata)])); + + assert!(synthesize_reused_result(&lockfile, &key, "pkg-from-tarball").is_none()); +} + +#[test] +fn does_not_reuse_a_package_absent_from_the_packages_map() { + let key: PkgNameVerPeer = "react@18.2.0".parse().expect("parse key"); + let lockfile = empty_lockfile(); + assert!(synthesize_reused_result(&lockfile, &key, "react").is_none()); +} + +#[test] +fn does_not_reuse_a_non_semver_version_slot() { + // A package keyed by a tarball URL has a non-semver version part; the + // peer-stripped metadata key still exists but `synthesize` bails + // before it because the version slot doesn't parse as a semver. + let key: PkgNameVerPeer = + "pkg@https://example.test/pkg.tgz".parse().expect("parse url-keyed entry"); + let mut lockfile = empty_lockfile(); + lockfile.packages = Some(HashMap::from([(key.clone(), registry_metadata())])); + assert!(synthesize_reused_result(&lockfile, &key, "pkg").is_none()); +} diff --git a/pacquet/crates/resolving-deps-resolver/src/resolve_dependency_tree.rs b/pacquet/crates/resolving-deps-resolver/src/resolve_dependency_tree.rs index df5df6cc47..56b4450e63 100644 --- a/pacquet/crates/resolving-deps-resolver/src/resolve_dependency_tree.rs +++ b/pacquet/crates/resolving-deps-resolver/src/resolve_dependency_tree.rs @@ -30,9 +30,51 @@ fn lock_recoverable(mutex: &Mutex) -> MutexGuard<'_, Inner> { } use crate::{ + lockfile_reuse::{reusable_importer_dep, synthesize_reused_result}, node_id::NodeId, resolved_tree::{DependenciesTreeNode, DirectDep, PeerDep, ResolvedPackage, ResolvedTree}, }; +use pacquet_lockfile::{PkgNameVerPeer, SnapshotEntry}; + +/// Which dependencies `pacquet update` excludes from lockfile-resolution +/// reuse. An excluded package re-resolves to highest-in-range, and its +/// whole subtree re-resolves with it (so the bump's new transitive deps +/// are picked up). Mirrors pnpm's `update` re-resolution scope. +#[derive(Default, Clone)] +pub enum UpdateReuseScope { + /// Reuse every still-satisfied dependency. `install` / `add`. + #[default] + All, + /// Reuse nothing — the whole graph re-resolves. `pacquet update` + /// with no selectors. + None, + /// Reuse everything except the named packages (matched at any depth). + /// `pacquet update `. + Except(std::collections::HashSet), +} + +/// How the current [`fn@resolve_node`] call may reuse the prior +/// lockfile's resolution instead of re-resolving from the registry. +/// +/// Threaded down the recursion to faithfully port pnpm's +/// `resolvedDependencies` / `parentPkg.updated` mechanism +/// (`resolveChildren` / `getDepsToResolve` in +/// [`resolveDependencies.ts`](https://github.com/pnpm/pnpm/blob/097983fbca/installing/deps-resolver/src/resolveDependencies.ts#L1000-L1248)). +#[derive(Clone)] +enum ReuseSource { + /// A direct dependency of importer `importer_id`. Reuse matches the + /// manifest specifier against the importer's recorded resolution via + /// semver-satisfies ([`reusable_importer_dep`]). + Importer { importer_id: String }, + /// A transitive dependency whose resolved snapshot key the parent's + /// snapshot already pins. `Some` reuses that key directly (no semver + /// check — the parent version pins it); `None` means an updated + /// ancestor discarded its child-refs, forcing this subtree to + /// re-resolve (pnpm's `parentPkg.updated ? undefined : refs`). + Transitive { key: Option }, + /// Reuse disabled for this node (no prior lockfile). + Off, +} /// Options threaded into [`fn@resolve_dependency_tree`]. /// @@ -206,7 +248,8 @@ where let injected = injected_names.contains(name); wanted.push((name.to_string(), range.to_string(), optional, injected)); } - let direct = extend_tree(&ctx, resolver, wanted).await?; + let direct = + extend_tree(&ctx, resolver, wanted, pacquet_lockfile::Lockfile::ROOT_IMPORTER_KEY).await?; Ok(ctx.into_resolved_tree(direct)) } @@ -372,6 +415,23 @@ pub struct WorkspaceTreeCtx { /// enters the wanted-dep cache. See [`ManifestHook`] for the /// signature. `None` when no hook is configured. manifest_hook: Option, + /// The previous `pnpm-lock.yaml` the install started from, when one + /// exists. Consulted by `resolve_node` to reuse an already-resolved + /// dependency + its transitive subtree instead of re-resolving from + /// the registry (see `pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md`). + /// `None` on a first install or when reuse is disabled. + wanted_lockfile: Option>, + /// Lockfile-reuse suppression for `pacquet update`. `update` + /// re-resolves its target deps to highest-in-range, so a reused + /// resolution would defeat the bump. Mirrors pnpm's `updateToLatest` + /// / `updateMatching` propagation into `parentPkg.updated`. See + /// [`UpdateReuseScope`]. + update_reuse_scope: UpdateReuseScope, + /// Memoises [`fn@subtree_fully_reusable`] per snapshot key so the + /// recursive reusability check runs once per package across the + /// whole walk. `true` means the package and its entire transitive + /// subtree can be synthesized from the prior lockfile. + subtree_reusable: Mutex>, } impl Default for WorkspaceTreeCtx { @@ -386,6 +446,9 @@ impl Default for WorkspaceTreeCtx { children_specs_by_id: Mutex::new(HashMap::new()), children_by_id: Mutex::new(HashMap::new()), manifest_hook: None, + wanted_lockfile: None, + update_reuse_scope: UpdateReuseScope::All, + subtree_reusable: Mutex::new(HashMap::new()), } } } @@ -416,6 +479,29 @@ impl WorkspaceTreeCtx { self } + /// Attach the prior `pnpm-lock.yaml` so `resolve_node` can reuse + /// already-resolved dependencies instead of re-resolving them. See + /// the `wanted_lockfile` field. + pub fn with_wanted_lockfile( + mut self, + wanted_lockfile: Option>, + ) -> Self { + self.wanted_lockfile = wanted_lockfile; + self + } + + /// The prior `pnpm-lock.yaml` to reuse resolutions from, if any. + pub fn wanted_lockfile(&self) -> Option<&Arc> { + self.wanted_lockfile.as_ref() + } + + /// Set which dependencies `pacquet update` excludes from reuse. See + /// [`UpdateReuseScope`]. + pub fn with_update_reuse_scope(mut self, scope: UpdateReuseScope) -> Self { + self.update_reuse_scope = scope; + self + } + /// Take ownership of `self` and emit the final [`ResolvedTree`]. /// Pacquet's single-importer path consumes the context via /// [`TreeCtx::into_resolved_tree`], which routes through here once @@ -629,32 +715,43 @@ pub async fn extend_tree( ctx: &TreeCtx, resolver: &Chain, wanted: Vec, + importer_id: &str, ) -> Result, ResolveDependencyTreeError> where Chain: Resolver + ?Sized, { + // Direct deps reuse via the importer's recorded resolution when a + // prior lockfile exists; without one the gate is a no-op. + let reuse = if ctx.workspace.wanted_lockfile.is_some() { + ReuseSource::Importer { importer_id: importer_id.to_string() } + } else { + ReuseSource::Off + }; let results = wanted .into_iter() - .map(|(name, range, optional, injected)| async move { - // `injected: Some(true)` only when the importer manifest's - // `dependenciesMeta[name].injected = true` opted this dep - // in. Otherwise leave it `None` — matches upstream's - // `injected: opts.dependenciesMeta[alias]?.injected` shape - // where an absent meta entry yields `undefined`, not - // `false`. The resolver OR's this with the global - // `inject_workspace_packages` flag, so `None` and - // `Some(false)` would produce identical behavior — but - // mirroring the upstream wire shape keeps the - // [`WantedKey`] cache buckets aligned across the two - // pacquet branches that surface `injected`. - let wanted = WantedDependency { - alias: Some(name), - bare_specifier: Some(range), - optional: Some(optional), - injected: injected.then_some(true), - ..WantedDependency::default() - }; - resolve_node(ctx, resolver, wanted, &[], 0, false).await + .map(|(name, range, optional, injected)| { + let reuse = reuse.clone(); + async move { + // `injected: Some(true)` only when the importer manifest's + // `dependenciesMeta[name].injected = true` opted this dep + // in. Otherwise leave it `None` — matches upstream's + // `injected: opts.dependenciesMeta[alias]?.injected` shape + // where an absent meta entry yields `undefined`, not + // `false`. The resolver OR's this with the global + // `inject_workspace_packages` flag, so `None` and + // `Some(false)` would produce identical behavior — but + // mirroring the upstream wire shape keeps the + // [`WantedKey`] cache buckets aligned across the two + // pacquet branches that surface `injected`. + let wanted = WantedDependency { + alias: Some(name), + bare_specifier: Some(range), + optional: Some(optional), + injected: injected.then_some(true), + ..WantedDependency::default() + }; + resolve_node(ctx, resolver, wanted, &[], 0, false, reuse).await + } }) .pipe(future::try_join_all) .await?; @@ -683,12 +780,35 @@ async fn resolve_node( ancestor_ids: &[String], depth: i32, parent_optional: bool, + reuse: ReuseSource, ) -> Result, ResolveDependencyTreeError> where Chain: Resolver + ?Sized, { let current_is_optional = wanted.optional.unwrap_or(false) || parent_optional; + // **Lockfile-resolution reuse.** When the prior lockfile already + // resolved this edge (and the recorded version still satisfies the + // manifest range, for a direct dep), synthesize the resolution from + // the lockfile and walk its transitive subtree from the snapshot + // graph instead of re-resolving from the registry. Mirrors pnpm's + // [`getInfoFromLockfile` reuse](https://github.com/pnpm/pnpm/blob/097983fbca/installing/deps-resolver/src/resolveDependencies.ts#L1199-L1248). + // `synthesize_reused_result` is conservative: any shape it can't + // faithfully reproduce (non-registry resolutions, missing metadata) + // yields `None` here and the node falls through to a fresh resolve. + if let Some(reused) = try_reuse_node(ctx, &wanted, reuse) { + return resolve_reused_node( + ctx, + resolver, + wanted, + ancestor_ids, + depth, + current_is_optional, + reused, + ) + .await; + } + // Memoise the per-wanted resolve. The first caller for a given // `(alias, bare_specifier, optional)` runs the resolver chain and // stores the `Arc` on `ctx.resolved_by_wanted`; @@ -909,6 +1029,10 @@ where }; let next_ancestors = next_ancestors.clone(); async move { + // A freshly-resolved node forces its whole subtree + // to re-resolve — pnpm's `resolvedDependencies = + // parentPkg.updated ? undefined`. `ReuseSource::Off` + // is the `undefined` arm. resolve_node( ctx, resolver, @@ -916,6 +1040,7 @@ where &next_ancestors, depth + 1, current_is_optional, + ReuseSource::Off, ) .await } @@ -975,6 +1100,325 @@ where Ok(Some(DirectDep { alias, node_id, id })) } +/// One reusable node: its prior-lockfile snapshot key plus the +/// `ResolveResult` synthesized from the lockfile metadata. +struct ReusedNode { + key: PkgNameVerPeer, + result: pacquet_resolving_resolver_base::ResolveResult, +} + +/// Decide whether the current edge can reuse the prior lockfile's +/// resolution. Returns the synthesized node when the edge's whole +/// transitive subtree is reusable; `None` (fresh resolve) otherwise. +/// +/// Conservative on every axis: no prior lockfile, an unsatisfied direct +/// range, a `link:` / non-registry shape anywhere in the subtree, or a +/// missing snapshot entry all yield `None`. See +/// [`fn@subtree_fully_reusable`] for the recursive subtree check. +fn try_reuse_node( + ctx: &TreeCtx, + wanted: &WantedDependency, + reuse: ReuseSource, +) -> Option { + let lockfile = ctx.workspace.wanted_lockfile.as_ref()?; + if matches!(ctx.workspace.update_reuse_scope, UpdateReuseScope::None) { + return None; + } + let alias = wanted.alias.as_deref()?; + let key = match reuse { + ReuseSource::Importer { importer_id } => { + let bare_specifier = wanted.bare_specifier.as_deref()?; + reusable_importer_dep(&lockfile.importers, &importer_id, alias, bare_specifier)? + } + ReuseSource::Transitive { key } => key?, + ReuseSource::Off => return None, + }; + if !subtree_fully_reusable(ctx, lockfile, &key) { + return None; + } + let result = synthesize_reused_result(lockfile, &key, alias)?; + Some(ReusedNode { key, result }) +} + +/// `true` when `name` is a `pacquet update` target excluded from reuse. +fn update_excludes(scope: &UpdateReuseScope, name: &pacquet_lockfile::PkgName) -> bool { + match scope { + UpdateReuseScope::All => false, + // `None` is handled earlier in `try_reuse_node`; treat it the + // same here for completeness. + UpdateReuseScope::None => true, + UpdateReuseScope::Except(names) => names.contains(&name.to_string()), + } +} + +/// `true` when `key` and its entire transitive subtree can be +/// synthesized from `lockfile` (every node a plain-semver registry +/// package present in `packages:`, every snapshot child non-`link:`). +/// Memoised on [`WorkspaceTreeCtx::subtree_reusable`] so each package is +/// checked once. +/// +/// A snapshot cycle is treated as **non**-reusable at the back-edge: the +/// key is provisionally inserted as `false` before recursing, so a node +/// reached through a still-in-progress ancestor resolves to `false` and +/// any subtree containing a dependency cycle conservatively re-resolves. +/// This avoids the unsound alternative — a provisional `true` could cache +/// a cycle member as reusable based on an ancestor that later finalizes +/// `false` (e.g. an update-excluded target reachable only through the +/// cycle), wrongly reusing it. SCC-aware reuse of acyclic-equivalent +/// cycles is possible but not worth the complexity for an uncommon case. +fn subtree_fully_reusable( + ctx: &TreeCtx, + lockfile: &pacquet_lockfile::Lockfile, + key: &PkgNameVerPeer, +) -> bool { + if let Some(&cached) = lock_recoverable(&ctx.workspace.subtree_reusable).get(key) { + return cached; + } + // Provisionally mark non-reusable so a cycle back to `key` resolves to + // `false` (re-resolve) instead of recursing forever — see the doc above + // for why `false` rather than `true`. + lock_recoverable(&ctx.workspace.subtree_reusable).insert(key.clone(), false); + // A `pacquet update` target anywhere in the subtree forces the whole + // subtree to re-resolve so the bump's new transitive deps are picked + // up — mirrors pnpm matching update names at any depth. + let reusable = !update_excludes(&ctx.workspace.update_reuse_scope, &key.name) + && synthesize_reused_result(lockfile, key, &key.name.to_string()).is_some() + && subtree_children_reusable(ctx, lockfile, key); + lock_recoverable(&ctx.workspace.subtree_reusable).insert(key.clone(), reusable); + reusable +} + +/// Recurse [`fn@subtree_fully_reusable`] across `key`'s snapshot +/// children. A `link:` child (no snapshot key) makes the subtree +/// non-reusable: the linked importer resolves its own deps, which this +/// reuse path doesn't model. +fn subtree_children_reusable( + ctx: &TreeCtx, + lockfile: &pacquet_lockfile::Lockfile, + key: &PkgNameVerPeer, +) -> bool { + let Some(snapshot) = lockfile.snapshots.as_ref().and_then(|snaps| snaps.get(key)) else { + // No snapshot entry → the lockfile doesn't record this node's + // children, so the reuse walk can't reproduce its subtree. + // Force a fresh resolve rather than risk silently dropping + // transitive deps. A genuine leaf has an empty-but-*present* + // snapshot entry (`{}`); a missing one means an inconsistent + // lockfile, which `try_reuse_node`'s contract sends to a fresh + // resolve. + return false; + }; + let dep_maps = [snapshot.dependencies.as_ref(), snapshot.optional_dependencies.as_ref()]; + for dep_map in dep_maps.into_iter().flatten() { + for (child_name, dep_ref) in dep_map { + let Some(child_key) = dep_ref.resolve(child_name) else { + return false; + }; + if !subtree_fully_reusable(ctx, lockfile, &child_key) { + return false; + } + } + } + true +} + +/// Register a node whose resolution was reused from the prior lockfile, +/// then walk its transitive children from the snapshot graph instead of +/// re-resolving them. Mirrors the post-resolve half of +/// [`fn@resolve_node`], specialized for a node whose subtree +/// [`fn@try_reuse_node`] already confirmed reusable. +#[async_recursion] +async fn resolve_reused_node( + ctx: &TreeCtx, + resolver: &Chain, + wanted: WantedDependency, + ancestor_ids: &[String], + depth: i32, + current_is_optional: bool, + reused: ReusedNode, +) -> Result, ResolveDependencyTreeError> +where + Chain: Resolver + ?Sized, +{ + let ReusedNode { key, result } = reused; + let result = Arc::new(result); + + // A reused node carries the synthesized registry resolution into the + // same per-wanted cache a fresh resolve would populate, so a later + // fresh-resolve of the identical wanted dep short-circuits to it. + let opts = ctx.opts_for_depth(depth); + let cache_key: WantedKey = ( + wanted.alias.clone(), + wanted.bare_specifier.clone(), + wanted.optional, + wanted.injected, + opts.pick_lowest_version, + opts.published_by, + ); + lock_recoverable(&ctx.workspace.resolved_by_wanted) + .entry(cache_key) + .or_insert_with(|| Arc::clone(&result)); + + let id = build_pkg_id_with_patch_hash(ctx, &result).await?; + + // Cycle break — same as the fresh path. + if ancestor_ids.iter().any(|prev| prev == &id) { + return Ok(None); + } + + let alias = result + .alias + .clone() + .or_else(|| wanted.alias.clone()) + .or_else(|| result.name_ver.as_ref().map(|nv| nv.name.to_string())) + .unwrap_or_else(|| id.clone()); + + // Leaf classification reads the snapshot graph (the source of truth + // for a reused node's children), not the synthesized manifest (whose + // `dependencies` are deliberately omitted). A node with no recorded + // children and no peers is a leaf, matching `pkg_is_leaf`. + let snapshot = ctx + .workspace + .wanted_lockfile + .as_ref() + .and_then(|lockfile| lockfile.snapshots.as_ref()) + .and_then(|snaps| snaps.get(&key)); + let child_refs = snapshot_child_refs(snapshot); + let peer_dependencies = extract_peer_dependencies(&result); + let is_leaf = child_refs.is_empty() && peer_dependencies.is_empty(); + let node_id = if is_leaf { NodeId::leaf(&id) } else { NodeId::next() }; + + let is_revisit; + { + let mut packages = lock_recoverable(&ctx.workspace.packages); + match packages.get_mut(&id) { + Some(existing) => { + existing.optional = existing.optional && current_is_optional; + is_revisit = true; + } + None => { + { + let mut all_peers = lock_recoverable(&ctx.workspace.all_peer_dep_names); + for name in peer_dependencies.keys() { + all_peers.insert(name.clone()); + } + } + packages.insert( + id.clone(), + ResolvedPackage { + id: id.clone(), + result: Arc::clone(&result), + peer_dependencies, + optional: current_is_optional, + is_leaf, + }, + ); + is_revisit = false; + } + } + } + + let next_ancestors: Vec = + ancestor_ids.iter().cloned().chain(std::iter::once(id.clone())).collect(); + + let children = if is_revisit { + crate::resolved_tree::TreeChildren::Lazy { parent_ids: Arc::new(next_ancestors.clone()) } + } else { + let child_results = child_refs + .iter() + .map(|(child_alias, child_key)| { + let child_wanted = WantedDependency { + alias: Some(child_alias.clone()), + // The snapshot pins the exact version; carry it as + // the bare specifier so the per-wanted dedup cache + // key is stable and a fresh fallback (if reuse were + // ever disabled) would still target the right pin. + bare_specifier: Some(child_key.suffix.without_peer().to_string()), + ..WantedDependency::default() + }; + let next_ancestors = next_ancestors.clone(); + let child_key = child_key.clone(); + async move { + resolve_node( + ctx, + resolver, + child_wanted, + &next_ancestors, + depth + 1, + current_is_optional, + ReuseSource::Transitive { key: Some(child_key) }, + ) + .await + } + }) + .pipe(future::try_join_all) + .await?; + let mut realized: BTreeMap = BTreeMap::new(); + let mut by_id: Vec = Vec::new(); + let optional_by_alias: HashMap<&str, bool> = child_refs + .iter() + .map(|(alias, _)| (alias.as_str(), is_optional_child(snapshot, alias))) + .collect(); + for dep in child_results.into_iter().flatten() { + let optional = optional_by_alias.get(dep.alias.as_str()).copied().unwrap_or(false); + by_id.push(crate::resolved_tree::ChildEdge { + alias: dep.alias.clone(), + pkg_id: dep.id.clone(), + optional, + }); + realized.insert(dep.alias, dep.node_id); + } + lock_recoverable(&ctx.workspace.children_by_id) + .entry(id.clone()) + .or_insert_with(|| Arc::new(by_id)); + crate::resolved_tree::TreeChildren::Realized(realized) + }; + + lock_recoverable(&ctx.workspace.dependencies_tree) + .entry(node_id.clone()) + .and_modify(|node| { + if node.depth > depth { + node.depth = depth; + } + }) + .or_insert_with(|| DependenciesTreeNode { + resolved_package_id: id.clone(), + children, + depth, + installable: true, + }); + + Ok(Some(DirectDep { alias, node_id, id })) +} + +/// `(install_alias, resolved_snapshot_key)` for every non-`link:` child +/// recorded on `snapshot`'s `dependencies` + `optionalDependencies`. +/// Sorted by alias so the per-occurrence walk order is deterministic. +fn snapshot_child_refs(snapshot: Option<&SnapshotEntry>) -> Vec<(String, PkgNameVerPeer)> { + let Some(snapshot) = snapshot else { return Vec::new() }; + let mut out: Vec<(String, PkgNameVerPeer)> = Vec::new(); + for dep_map in [snapshot.dependencies.as_ref(), snapshot.optional_dependencies.as_ref()] + .into_iter() + .flatten() + { + for (alias, dep_ref) in dep_map { + if let Some(key) = dep_ref.resolve(alias) { + out.push((alias.to_string(), key)); + } + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +/// `true` when `alias` is recorded under `snapshot.optionalDependencies` +/// (as opposed to `dependencies`). Threads the right `optional` flag onto +/// the reused child's [`crate::resolved_tree::ChildEdge`]. +fn is_optional_child(snapshot: Option<&SnapshotEntry>, alias: &str) -> bool { + let Some(snapshot) = snapshot else { return false }; + let Ok(name) = alias.parse::() else { return false }; + snapshot.optional_dependencies.as_ref().is_some_and(|deps| deps.contains_key(&name)) +} + /// Replace `catalog:` bare specifiers on direct dependencies with the /// version recorded in the catalogs map. Non-`catalog:` specifiers /// pass through unchanged. diff --git a/pacquet/crates/resolving-deps-resolver/src/resolve_importer.rs b/pacquet/crates/resolving-deps-resolver/src/resolve_importer.rs index 67acce07d9..8ab4236119 100644 --- a/pacquet/crates/resolving-deps-resolver/src/resolve_importer.rs +++ b/pacquet/crates/resolving-deps-resolver/src/resolve_importer.rs @@ -214,7 +214,15 @@ where // and can't mutate it after the fact. let workspace = Arc::new(WorkspaceTreeCtx::default().with_manifest_hook(opts.manifest_hook.clone())); - resolve_importer_with_workspace(resolver, manifest, dependency_groups, opts, workspace).await + resolve_importer_with_workspace( + resolver, + pacquet_lockfile::Lockfile::ROOT_IMPORTER_KEY, + manifest, + dependency_groups, + opts, + workspace, + ) + .await } /// Same as [`fn@resolve_importer`] but reuses a shared @@ -225,6 +233,7 @@ where /// map. pub async fn resolve_importer_with_workspace( resolver: &Chain, + importer_id: &str, manifest: &PackageManifest, dependency_groups: DependencyGroupList, opts: ResolveImporterOptions, @@ -269,7 +278,7 @@ where let initial_wanted = importer_direct_wanted_specs(manifest, dependency_groups, auto_install_peers, &catalogs)?; - let mut direct = extend_tree(&ctx, resolver, initial_wanted).await?; + let mut direct = extend_tree(&ctx, resolver, initial_wanted, importer_id).await?; update_preferred_versions_with_ctx(&ctx, &mut all_preferred_versions); let mut parent_pkg_aliases: HashSet = @@ -335,7 +344,7 @@ where // threading the per-dep meta. let new_wanted: Vec = hoisted.into_iter().map(|(name, range)| (name, range, false, false)).collect(); - let new_direct = extend_tree(&ctx, resolver, new_wanted).await?; + let new_direct = extend_tree(&ctx, resolver, new_wanted, importer_id).await?; direct.extend(new_direct); update_preferred_versions_with_ctx(&ctx, &mut all_preferred_versions); } @@ -358,7 +367,7 @@ where // also defaults to `false` for the same reason. let new_wanted: Vec = hoisted_optional.into_iter().map(|(name, range)| (name, range, false, false)).collect(); - let new_direct = extend_tree(&ctx, resolver, new_wanted).await?; + let new_direct = extend_tree(&ctx, resolver, new_wanted, importer_id).await?; direct.extend(new_direct); update_preferred_versions_with_ctx(&ctx, &mut all_preferred_versions); all_missing_optional_peers.clear(); diff --git a/pacquet/crates/resolving-deps-resolver/src/resolve_workspace.rs b/pacquet/crates/resolving-deps-resolver/src/resolve_workspace.rs index 3a8c00d0db..7c3f55a725 100644 --- a/pacquet/crates/resolving-deps-resolver/src/resolve_workspace.rs +++ b/pacquet/crates/resolving-deps-resolver/src/resolve_workspace.rs @@ -18,7 +18,9 @@ //! resolved-pkgs share is a follow-up perf win. use crate::{ - resolve_dependency_tree::{ManifestHook, WorkspaceTreeCtx, importer_direct_wanted_specs}, + resolve_dependency_tree::{ + ManifestHook, UpdateReuseScope, WorkspaceTreeCtx, importer_direct_wanted_specs, + }, resolve_importer::{ ResolveImporterError, ResolveImporterOptions, ResolveImporterResult, resolve_importer_with_workspace, @@ -76,6 +78,17 @@ pub struct WorkspaceResolveOptions { /// [`getPublishedByDate`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/installing/deps-resolver/src/resolveDependencies.ts#L506-L517) /// step. pub time_based: bool, + + /// The prior `pnpm-lock.yaml` the install started from, when one + /// exists. Threaded into [`WorkspaceTreeCtx`] so the tree walk can + /// reuse already-resolved dependencies instead of re-resolving them + /// (see `pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md`). `None` on a + /// first install or when reuse is disabled. + pub wanted_lockfile: Option>, + + /// Which dependencies `pacquet update` excludes from lockfile- + /// resolution reuse. [`UpdateReuseScope::All`] for `install` / `add`. + pub update_reuse_scope: UpdateReuseScope, } /// Result of [`fn@resolve_workspace`]. The combined @@ -117,8 +130,15 @@ where manifest_hook, pick_lowest_direct, time_based, + wanted_lockfile, + update_reuse_scope, } = opts; - let workspace = Arc::new(WorkspaceTreeCtx::default().with_manifest_hook(manifest_hook)); + let workspace = Arc::new( + WorkspaceTreeCtx::default() + .with_manifest_hook(manifest_hook) + .with_wanted_lockfile(wanted_lockfile) + .with_update_reuse_scope(update_reuse_scope), + ); // Build every importer's options up front so the `time-based` // pre-pass and the resolve loop see the same per-importer wiring. @@ -152,6 +172,7 @@ where let modules_dir = importer_opts.modules_dir.clone(); let ResolveImporterResult { resolved_tree, .. } = resolve_importer_with_workspace( resolver, + &importer.id, importer.manifest, dependency_groups.iter().copied(), importer_opts, diff --git a/pacquet/crates/resolving-deps-resolver/src/resolve_workspace/tests.rs b/pacquet/crates/resolving-deps-resolver/src/resolve_workspace/tests.rs index b9ae08e993..68de4dc613 100644 --- a/pacquet/crates/resolving-deps-resolver/src/resolve_workspace/tests.rs +++ b/pacquet/crates/resolving-deps-resolver/src/resolve_workspace/tests.rs @@ -131,6 +131,8 @@ fn workspace_opts(pick_lowest_direct: bool, time_based: bool) -> WorkspaceResolv manifest_hook: None, pick_lowest_direct, time_based, + wanted_lockfile: None, + update_reuse_scope: crate::UpdateReuseScope::All, } } diff --git a/pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md b/pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md new file mode 100644 index 0000000000..dd8e216fee --- /dev/null +++ b/pacquet/plans/LOCKFILE_RESOLUTION_REUSE.md @@ -0,0 +1,114 @@ +# Lockfile-resolution reuse (pacquet) + +Port pnpm's behavior: during a non-frozen install, **reuse the prior lockfile's +resolution + transitive subtree** for dependencies that are still satisfied and +not being updated, instead of re-resolving everything from manifests (pacquet +today only feeds the lockfile into preferred-versions seeding). This closes the +perf gap that the merged tarball warm-store-reuse PR (#12096) only patched for +remote tarballs, and matches how pnpm avoids re-resolving unchanged trees. + +## pnpm reference (source of truth) + +`installing/deps-resolver/src/resolveDependencies.ts`: +- `getInfoFromLockfile(lockfile, registries, reference, alias)` (~L1199) — look up + the recorded snapshot for an alias's ref; returns resolution + `dependencyLockfile` + (the transitive child refs). +- Reuse predicate in `resolveDependenciesOfDependency` (~L844-881): `update = false` + unless update-requested, the snapshot is missing (new dep), a workspace pkg became + available, or the parent is in `updatedSet`. +- `getDepsToResolve` (~L1086) matches each wanted child against `resolvedDependencies[alias]` + via `satisfiesWanted` (semver-satisfies, not string-equality). +- Subtree propagation in `resolveChildren` (~L1000): `resolvedDependencies = + parentPkg.updated ? undefined : currentResolvedDependencies` — an unchanged parent + feeds its lockfile child-refs down; an updated parent discards them, forcing the + whole subtree to re-resolve. +- `packageRequester.ts` (~L155-277): on `update=false` the request returns + `updated:false` and skips fetch. + +## Key simplification for pacquet + +A given package **version's dependency set is immutable**, and the lockfile snapshot +already reflects any `readPackageHook`/`packageExtensions` that were applied when it +was written. So for a reused parent version, its transitive subtree is exactly the +snapshot's recorded child-refs — we can **walk the snapshot subtree (frozen-install +style) instead of re-resolving from the parent manifest**, and need neither the +parent's package.json nor its child *ranges*. `install_frozen_lockfile.rs` already +performs this snapshot→graph walk and is the reusable building block. + +A *changed* `readPackageHook`/`packageExtensions` config invalidates reuse: the install +withholds the prior lockfile from the reuse path when its `packageExtensionsChecksum` no +longer matches the config, so the stale subtree is re-resolved (mirrors pnpm invalidating +the lockfile on a settings change). `overrides` drift is not yet guarded for transitive +reuse — see follow-ups. + +## Design: hybrid resolve + +Fresh-resolve new/changed/update-targeted deps + their subtrees through the existing +`resolve_node` path; snapshot-walk the unchanged subtrees; merge into one +`DependenciesGraph`. The reuse decision threads down the recursion exactly like pnpm's +`resolvedDependencies`. + +### Stage 1 — plumbing +Thread `wanted_lockfile: Option>` from +`install_with_fresh_lockfile.rs` → `resolve_workspace` → `resolve_importer` → +`WorkspaceTreeCtx` (`resolve_dependency_tree.rs`). Also thread the active +`UpdateSeedPolicy` so the gate can suppress reuse for update-targeted names. +(Lands together with Stage 2 — an unused field would trip `-D warnings`.) + +### Stage 2 — reuse gate (semver-satisfies) +Add a recursion parameter carrying the lockfile child-refs for the current subtree +(`Option<&BTreeMap>`); at importer level it comes from +`lockfile.importers[id]` (`ProjectSnapshot.dependencies` + `.specifiers`). +In `resolve_node`, before the resolver call, compute a `reference`: +- importer dep: reuse only when the manifest specifier **semver-satisfies** the + recorded version (`node-semver`), the dep isn't update-targeted, and the + snapshot+package entry exist. +- transitive dep: take the ref from the passed-down child-refs map. +When matched, synthesize the `ResolveResult` from the lockfile (`PackageMetadata` +resolution + integrity; manifest reconstructed from the snapshot / read from the +store-index bundled manifest as the tarball-reuse path does) and skip the resolver. +Children still resolve normally in this stage. + +### Stage 3 — subtree reuse (the real win) +When a node is reused and not update-propagated, build its children from the +snapshot's dep-refs (reuse `install_frozen_lockfile`'s walk) instead of +`extract_children` + recursion. Carry an `updated` flag down so an updated ancestor +discards the child-refs (passes `None`) and forces its subtree to re-resolve — +faithful to `parentPkg.updated ? undefined : refs`. + +### Stage 4 — update suppression +Wire `UpdateSeedPolicy` (KeepAll / DropAll / DropOnly) into the gate so +`pacquet update [selector]` / `--latest` bypasses reuse for targeted deps and +propagates down their subtrees. + +### Stage 5 — tests + benchmark +- Port pnpm's reuse/update suites (`resolveDependencies`, `install/update.ts`) as + Rust tests first (per the "port tests before optimizations" rule). +- Discriminating no-re-resolve test: mockito server + dead-server, like #12096 — a + second install with an unchanged dep must succeed with the registry down. +- Peer correctness: verify against the ported peer tests (pacquet's separate peer + pass is the subtlest interaction). +- vlt.sh before/after on a deep-transitive fixture for the perf number. + +## Risk +Stage 3 is high-blast-radius: wrong reuse → wrong tree → wrong installs. The peer +pass and the `updated`-propagation boundary are the subtlest parts. + +## Known follow-ups (before un-drafting) + +- **Lockfile byte-ordering is build-order-dependent** ([#12117](https://github.com/pnpm/pnpm/issues/12117)). Surfaced by the + structural-equivalence test: reuse and fresh resolves produce + *content-identical* lockfiles, but the writer emits the + `packages` / `snapshots` / importer-`dependencies` maps in + build-insertion order, so a re-install can reorder the lockfile (spurious + git diff) even though nothing changed. pnpm emits these canonically + sorted. Likely a pre-existing writer gap that reuse makes user-visible; + fix by sorting those maps at emit time so lockfiles are byte-stable. +- **`overrides` drift** isn't yet guarded for transitive reuse (only + `packageExtensions` is). An `overrides` change that rewrites a transitive + dep's version should invalidate that subtree's reuse. +- **Dependency cycles conservatively re-resolve.** `subtree_fully_reusable` treats a + still-in-progress back-edge as non-reusable, so any subtree containing a cycle is + re-resolved rather than reused (correct, but a perf limitation). SCC-aware reuse of + acyclic-equivalent cycles is a possible future optimization. +- vlt.sh before/after benchmark for the perf number.