diff --git a/.changeset/fast-exact-overrides.md b/.changeset/fast-exact-overrides.md new file mode 100644 index 0000000000..5befeab404 --- /dev/null +++ b/.changeset/fast-exact-overrides.md @@ -0,0 +1,9 @@ +--- +"@pnpm/installing.deps-installer": patch +"@pnpm/installing.deps-restorer": patch +"@pnpm/installing.linking.modules-cleaner": patch +"pnpm": patch +"pacquet": patch +--- + +Speed up installs after safe override changes by reusing unambiguous compatible dependency resolutions, pruning obsolete dependencies, applying independent replacements and removals together, and handling parent-scoped `"-"` overrides without full lockfile resolution. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b52352cc29..fa4ff2c080 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6428,6 +6428,9 @@ importers: ramda: specifier: 'catalog:' version: '@pnpm/ramda@0.28.1' + validate-npm-package-name: + specifier: 'catalog:' + version: 7.0.2 devDependencies: '@pnpm/installing.linking.modules-cleaner': specifier: workspace:* @@ -6438,6 +6441,9 @@ importers: '@types/ramda': specifier: 'catalog:' version: 0.32.0 + '@types/validate-npm-package-name': + specifier: 'catalog:' + version: 4.0.2 pnpm11/installing/linking/real-hoist: dependencies: diff --git a/pnpm/crates/cli/tests/lockfile_resolution_reuse.rs b/pnpm/crates/cli/tests/lockfile_resolution_reuse.rs index dfec5322d2..582e0682dc 100644 --- a/pnpm/crates/cli/tests/lockfile_resolution_reuse.rs +++ b/pnpm/crates/cli/tests/lockfile_resolution_reuse.rs @@ -29,6 +29,162 @@ fn dead_registry_url() -> String { format!("http://127.0.0.1:{}/", addr.port()) } +#[test] +fn exact_override_update_reuses_the_locked_children() { + let fixture = CommandTempCwd::init().add_mocked_registry(); + let manifest_path = fixture.workspace.join("package.json"); + let workspace_yaml_path = fixture.workspace.join("pnpm-workspace.yaml"); + fs::write( + &manifest_path, + serde_json::json!({ + "dependencies": { + "@pnpm.e2e/parent-of-pkg-with-1-dep": "1.0.0" + } + }) + .to_string(), + ) + .expect("write package.json"); + let workspace_yaml = + fs::read_to_string(&workspace_yaml_path).expect("read pnpm-workspace.yaml"); + fs::write( + &workspace_yaml_path, + format!("{workspace_yaml}overrides:\n '@pnpm.e2e/pkg-with-1-dep': 100.0.0\n"), + ) + .expect("write initial override"); + pacquet_at(&fixture.workspace).with_arg("install").assert().success(); + + let before = pacquet_lockfile::Lockfile::load_wanted_from_dir(&fixture.workspace) + .expect("load wanted lockfile") + .expect("wanted lockfile"); + let old_key = "@pnpm.e2e/pkg-with-1-dep@100.0.0".parse().expect("old key"); + let child_name = "@pnpm.e2e/dep-of-pkg-with-1-dep".parse().expect("child name"); + let old_child = before + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&old_key)) + .and_then(|snapshot| snapshot.dependencies.as_ref()) + .and_then(|dependencies| dependencies.get(&child_name)) + .cloned() + .expect("old locked child"); + + let workspace_yaml = fs::read_to_string(&workspace_yaml_path).expect("read initial override"); + fs::write( + &workspace_yaml_path, + workspace_yaml + .replace("'@pnpm.e2e/pkg-with-1-dep': 100.0.0", "'@pnpm.e2e/pkg-with-1-dep': 100.1.0"), + ) + .expect("update exact override"); + pacquet_at(&fixture.workspace).with_arg("install").assert().success(); + + let wanted = pacquet_lockfile::Lockfile::load_wanted_from_dir(&fixture.workspace) + .expect("load updated wanted lockfile") + .expect("updated wanted lockfile"); + let current = pacquet_lockfile::Lockfile::load_current_from_virtual_store_dir( + &fixture.workspace.join("node_modules/.pnpm"), + ) + .expect("load current lockfile") + .expect("current lockfile"); + let new_key = "@pnpm.e2e/pkg-with-1-dep@100.1.0".parse().expect("new key"); + for lockfile in [&wanted, ¤t] { + assert_eq!( + lockfile + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&new_key)) + .and_then(|snapshot| snapshot.dependencies.as_ref()) + .and_then(|dependencies| dependencies.get(&child_name)), + Some(&old_child), + ); + assert!( + lockfile.snapshots.as_ref().is_some_and(|snapshots| !snapshots.contains_key(&old_key)), + ); + } + + drop(fixture); +} + +#[test] +fn dependency_removal_override_prunes_the_locked_subtree_without_resolving() { + let CommandTempCwd { workspace, root, npmrc_info, .. } = + CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { mock_instance, npmrc_path, .. } = npmrc_info; + let manifest_path = workspace.join("package.json"); + let workspace_yaml_path = workspace.join("pnpm-workspace.yaml"); + fs::write( + &manifest_path, + serde_json::json!({ + "dependencies": { + "@pnpm.e2e/pkg-with-good-optional": "1.0.0" + } + }) + .to_string(), + ) + .expect("write package.json"); + let workspace_yaml = + fs::read_to_string(&workspace_yaml_path).expect("read pnpm-workspace.yaml"); + fs::write(&workspace_yaml_path, format!("{workspace_yaml}trustLockfile: true\n")) + .expect("enable trusted lockfile"); + pacquet_at(&workspace).with_arg("install").assert().success(); + + let workspace_yaml = + fs::read_to_string(&workspace_yaml_path).expect("read pnpm-workspace.yaml"); + fs::write(&workspace_yaml_path, format!("{workspace_yaml}overrides:\n is-positive: '-'\n")) + .expect("add dependency removal override"); + let dead_registry = dead_registry_url(); + let npmrc = fs::read_to_string(&npmrc_path).expect("read .npmrc"); + let npmrc = npmrc + .lines() + .filter(|line| !line.trim_start().starts_with("registry=")) + .collect::>() + .join("\n"); + fs::write(&npmrc_path, format!("registry={dead_registry}\n{npmrc}\n")) + .expect("rewrite .npmrc with a dead registry"); + + pacquet_at(&workspace).with_arg("install").assert().success(); + + let wanted = pacquet_lockfile::Lockfile::load_wanted_from_dir(&workspace) + .expect("load updated wanted lockfile") + .expect("updated wanted lockfile"); + let current = pacquet_lockfile::Lockfile::load_current_from_virtual_store_dir( + &workspace.join("node_modules/.pnpm"), + ) + .expect("load current lockfile") + .expect("current lockfile"); + let parent_key = "@pnpm.e2e/pkg-with-good-optional@1.0.0".parse().expect("parent package key"); + let removed_key = "is-positive@1.0.0".parse().expect("removed package key"); + let removed_name = "is-positive".parse().expect("removed package name"); + for lockfile in [&wanted, ¤t] { + dbg!(&lockfile.snapshots, &lockfile.packages); + assert!( + lockfile + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&parent_key)) + .and_then(|snapshot| snapshot.optional_dependencies.as_ref()) + .is_none_or(|dependencies| !dependencies.contains_key(&removed_name)), + ); + assert!( + lockfile + .snapshots + .as_ref() + .is_none_or(|snapshots| !snapshots.contains_key(&removed_key)), + ); + assert!( + lockfile.packages.as_ref().is_none_or(|packages| !packages.contains_key(&removed_key)), + ); + } + dbg!(&workspace); + assert!( + !workspace + .join( + "node_modules/.pnpm/@pnpm.e2e+pkg-with-good-optional@1.0.0/node_modules/is-positive", + ) + .exists(), + ); + + drop((root, mock_instance)); +} + #[test] fn reuses_unchanged_subtree_without_re_resolving_from_the_registry() { let CommandTempCwd { workspace, root, npmrc_info, .. } = diff --git a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs index 4a6148a291..ab8fe73f9a 100644 --- a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs +++ b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs @@ -688,17 +688,21 @@ fn read_string_or_list( } } -/// `Some(true)` when the manifest declares a `bin` entry (string or -/// non-empty object map), recorded as the `hasBin: true` signal; the -/// field is dropped entirely when absent. -fn manifest_has_bin(manifest: Option<&Value>) -> Option { - let value = manifest?.get("bin")?; - let present = match value { +/// `Some(true)` when the manifest declares executable files, recorded as +/// the `hasBin: true` signal; the field is dropped entirely when absent. +pub(crate) fn manifest_has_bin(manifest: Option<&Value>) -> Option { + let manifest = manifest?; + let has_bin = manifest.get("bin").is_some_and(|value| match value { Value::String(s) => !s.is_empty(), Value::Object(map) => !map.is_empty(), _ => false, - }; - present.then_some(true) + }); + let has_bin_directory = manifest + .get("directories") + .and_then(Value::as_object) + .and_then(|directories| directories.get("bin")) + .is_some_and(|value| value.as_str().is_some_and(|path| !path.is_empty())); + (has_bin || has_bin_directory).then_some(true) } /// Returned `Option`-pair from [`build_peer_dep_blocks`]: the diff --git a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile/tests.rs b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile/tests.rs index 9909beb402..049e23cd86 100644 --- a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile/tests.rs +++ b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile/tests.rs @@ -1,6 +1,7 @@ use super::{ DependenciesGraphToLockfileError, GraphToLockfileOptions, ImporterLockfileInput, - dependencies_graph_to_lockfile as try_dependencies_graph_to_lockfile, read_string_or_list, + dependencies_graph_to_lockfile as try_dependencies_graph_to_lockfile, manifest_has_bin, + read_string_or_list, }; use indexmap::IndexMap; use pacquet_deps_path::DepPath; @@ -25,6 +26,19 @@ use std::{ use tempfile::TempDir; use text_block_macros::text_block; +#[test] +fn recognizes_bin_directories_in_package_manifests() { + assert_eq!( + manifest_has_bin(Some(&json!({ + "directories": { + "bin": "cli" + } + }))), + Some(true), + ); + assert_eq!(manifest_has_bin(Some(&json!({ "directories": { "bin": "" } }))), None); +} + fn dependencies_graph_to_lockfile(opts: GraphToLockfileOptions<'_>) -> pacquet_lockfile::Lockfile { try_dependencies_graph_to_lockfile(opts).expect("convert dependency graph to lockfile") } diff --git a/pnpm/crates/package-manager/src/fast_update_overrides.rs b/pnpm/crates/package-manager/src/fast_update_overrides.rs new file mode 100644 index 0000000000..ce813d149f --- /dev/null +++ b/pnpm/crates/package-manager/src/fast_update_overrides.rs @@ -0,0 +1,619 @@ +use futures_util::future::join_all; +use indexmap::IndexMap; +use node_semver::{Range, Version}; +use pacquet_config_parse_overrides::{PackageSelector, VersionOverride}; +use pacquet_lockfile::{ + BundledDependencies, ImporterDepVersion, Lockfile, LockfileResolution, PackageKey, + PackageMetadata, PkgName, PkgNameVerPeer, PkgVerPeer, Prefix, ResolvedDependencyMap, + SnapshotDepRef, SnapshotEntry, StringOrList, pick_registry_for_package, +}; +use pacquet_resolving_deps_resolver::ManifestHook; +use pacquet_resolving_resolver_base::{ResolveOptions, ResolveResult, Resolver, WantedDependency}; +use serde_json::Value; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + sync::Arc, +}; + +struct FastOverride { + name: PkgName, + new_version: Option, + old_version: Option, + parent: Option, +} + +struct RewritePlan { + overrides: Vec, + peer_names: HashSet, + replacements: HashMap, +} + +struct ResolvedOverride { + manifest: Arc, + resolution: LockfileResolution, +} + +pub(crate) struct FastOverrideOptions<'a> { + pub lockfile: &'a Lockfile, + pub parsed_overrides: &'a [VersionOverride], + pub resolved_overrides: &'a IndexMap, + pub resolver: &'a dyn Resolver, + pub resolve_options: &'a ResolveOptions, + pub manifest_hook: Option<&'a ManifestHook>, + pub registries: &'a HashMap, + pub lockfile_include_tarball_url: bool, +} + +pub(crate) async fn try_fast_update_overrides(opts: FastOverrideOptions<'_>) -> Option { + let plan = build_rewrite_plan(opts.lockfile, opts.parsed_overrides, opts.resolved_overrides)?; + let resolutions = join_all( + plan.overrides + .iter() + .filter(|override_entry| { + override_entry.new_version.is_some() + && plan + .replacements + .iter() + .any(|(old, new)| old != new && old.name == override_entry.name) + }) + .map(|override_entry| resolve_override(&opts, override_entry)), + ) + .await; + let resolved: HashMap<_, _> = + resolutions.into_iter().collect::>>()?.into_iter().collect(); + rewrite_lockfile(&opts, &plan, &resolved) +} + +async fn resolve_override( + opts: &FastOverrideOptions<'_>, + override_entry: &FastOverride, +) -> Option<(PkgName, ResolvedOverride)> { + let name = override_entry.name.to_string(); + let version = override_entry.new_version.as_ref()?.to_string(); + let wanted = WantedDependency { + alias: Some(name.clone()), + bare_specifier: Some(version.clone()), + ..WantedDependency::default() + }; + let result = opts.resolver.resolve(&wanted, opts.resolve_options).await.ok()??; + let manifest = result.manifest.as_ref().map(Arc::clone)?; + let manifest = match opts.manifest_hook { + Some(hook) => hook(manifest), + None => manifest, + }; + if !is_safe_registry_result(&result, &manifest, &name, &version) { + return None; + } + Some(( + override_entry.name.clone(), + ResolvedOverride { manifest, resolution: result.resolution }, + )) +} + +fn build_rewrite_plan( + lockfile: &Lockfile, + parsed_overrides: &[VersionOverride], + resolved_overrides: &IndexMap, +) -> Option { + let old_overrides = lockfile.overrides.as_ref(); + if old_overrides + .is_some_and(|old| old.keys().any(|selector| !resolved_overrides.contains_key(selector))) + { + return None; + } + let parsed_by_selector: HashMap<&str, &VersionOverride> = + parsed_overrides.iter().map(|entry| (entry.selector.as_str(), entry)).collect(); + let mut overrides = Vec::new(); + for (selector, new_value) in resolved_overrides { + let old_value = old_overrides.and_then(|old| old.get(selector)); + if old_value == Some(new_value) { + continue; + } + let parsed = parsed_by_selector.get(selector.as_str())?; + let removes_dependency = new_value == "-"; + if parsed.target_pkg.bare_specifier.is_some() + || parsed.converge + || !removes_dependency && parsed.parent_pkg.is_some() + || parsed_overrides.iter().any(|candidate| { + candidate.selector != *selector + && candidate.target_pkg.name == parsed.target_pkg.name + }) + { + return None; + } + let name = PkgName::parse(&parsed.target_pkg.name).ok()?; + if overrides.iter().any(|entry: &FastOverride| entry.name == name) { + return None; + } + let new_version = + if removes_dependency { None } else { Some(Version::parse(new_value).ok()?) }; + overrides.push(FastOverride { + name, + new_version, + old_version: match (removes_dependency, old_value) { + (true, _) => None, + (false, Some(value)) => Some(Version::parse(value).ok()?), + (false, None) => None, + }, + parent: parsed.parent_pkg.clone(), + }); + } + if overrides.is_empty() { + return None; + } + + let peer_names = get_peer_names(lockfile); + if overrides + .iter() + .filter(|entry| entry.new_version.is_none()) + .any(|entry| peer_names.contains(&entry.name)) + { + return None; + } + + let by_name: HashMap<&PkgName, &FastOverride> = overrides + .iter() + .filter(|entry| entry.new_version.is_some()) + .map(|entry| (&entry.name, entry)) + .collect(); + let mut replacements = HashMap::new(); + for (alias, key) in all_dependency_keys(lockfile) { + let Some(override_entry) = by_name.get(alias) else { continue }; + let key = key?; + if key.name != *alias + || !key.suffix.peer().is_empty() + || key.suffix.prefix() != Prefix::None + || override_entry + .old_version + .as_ref() + .is_some_and(|old| key.suffix.version_semver() != Some(old)) + { + return None; + } + let old_snapshot = lockfile.snapshots.as_ref()?.get(&key)?; + let old_metadata = lockfile.packages.as_ref()?.get(&key.without_peer())?; + let safe_resolution = matches!(old_metadata.resolution, LockfileResolution::Registry(_)) + || matches!( + old_metadata.resolution, + LockfileResolution::Tarball(ref tarball) + if tarball.integrity.is_some() && tarball.git_hosted != Some(true), + ); + if old_snapshot.optional + || old_snapshot.patched == Some(true) + || old_snapshot.id.is_some() + || old_metadata.peer_dependencies.is_some() + || old_metadata.peer_dependencies_meta.is_some() + || !safe_resolution + { + return None; + } + let new_suffix: PkgVerPeer = + override_entry.new_version.as_ref()?.to_string().parse().ok()?; + replacements.insert(key, PkgNameVerPeer::new(alias.clone(), new_suffix)); + } + for (alias, key) in all_dependency_keys(lockfile) { + if key.is_some_and(|key| replacements.contains_key(&key)) && !by_name.contains_key(alias) { + return None; + } + } + Some(RewritePlan { overrides, peer_names, replacements }) +} + +fn get_peer_names(lockfile: &Lockfile) -> HashSet { + let mut result = HashSet::new(); + for metadata in lockfile.packages.as_ref().into_iter().flat_map(|map| map.values()) { + for name in metadata.peer_dependencies.as_ref().into_iter().flat_map(|map| map.keys()) { + if let Ok(name) = PkgName::parse(name) { + result.insert(name); + } + } + for name in metadata.peer_dependencies_meta.as_ref().into_iter().flat_map(|map| map.keys()) + { + if let Ok(name) = PkgName::parse(name) { + result.insert(name); + } + } + } + for snapshot in lockfile.snapshots.as_ref().into_iter().flat_map(|map| map.values()) { + for name in snapshot.transitive_peer_dependencies.iter().flatten() { + if let Ok(name) = PkgName::parse(name) { + result.insert(name); + } + } + } + result +} + +fn all_dependency_keys(lockfile: &Lockfile) -> Vec<(&PkgName, Option)> { + let mut result = Vec::new(); + for importer in lockfile.importers.values() { + for dependencies in [ + importer.dependencies.as_ref(), + importer.dev_dependencies.as_ref(), + importer.optional_dependencies.as_ref(), + ] + .into_iter() + .flatten() + { + for (alias, spec) in dependencies { + result.push((alias, spec.version.resolved_key(alias))); + } + } + } + for snapshot in lockfile.snapshots.as_ref().into_iter().flat_map(|map| map.values()) { + for dependencies in + [snapshot.dependencies.as_ref(), snapshot.optional_dependencies.as_ref()] + .into_iter() + .flatten() + { + for (alias, dep_ref) in dependencies { + result.push((alias, dep_ref.resolve(alias))); + } + } + } + result +} + +fn is_safe_registry_result( + result: &ResolveResult, + manifest: &Value, + name: &str, + version: &str, +) -> bool { + result.resolved_via == "npm-registry" + && result.policy_violation.is_none() + && result.name_ver.as_ref().is_some_and(|name_ver| { + name_ver.name.to_string() == name && name_ver.suffix.to_string() == version + }) + && manifest.get("name").and_then(Value::as_str) == Some(name) + && manifest.get("version").and_then(Value::as_str) == Some(version) + && manifest + .get("peerDependencies") + .is_none_or(|value| value.as_object().is_some_and(serde_json::Map::is_empty)) + && manifest + .get("peerDependenciesMeta") + .is_none_or(|value| value.as_object().is_some_and(serde_json::Map::is_empty)) + && manifest.get("deprecated").is_none() + && manifest.get("bundledDependencies").is_none() + && manifest.get("bundleDependencies").is_none() + && manifest.get("engines").is_none_or(|value| { + value.as_object().is_some_and(|engines| !engines.contains_key("runtime")) + }) + && matches!( + result.resolution, + LockfileResolution::Tarball(ref tarball) + if tarball.integrity.is_some() && tarball.git_hosted != Some(true), + ) +} + +fn rewrite_lockfile( + opts: &FastOverrideOptions<'_>, + plan: &RewritePlan, + resolved: &HashMap, +) -> Option { + let mut updated = opts.lockfile.clone(); + for importer in updated.importers.values_mut() { + rewrite_importer_dependencies(&mut importer.dependencies, plan); + rewrite_importer_dependencies(&mut importer.dev_dependencies, plan); + rewrite_importer_dependencies(&mut importer.optional_dependencies, plan); + } + let original_snapshots = opts.lockfile.snapshots.as_ref()?; + let mut snapshots = original_snapshots.clone(); + let mut packages = opts.lockfile.packages.clone()?; + for (key, snapshot) in &mut snapshots { + rewrite_snapshot_dependencies(&mut snapshot.dependencies, plan, Some(key)); + rewrite_snapshot_dependencies(&mut snapshot.optional_dependencies, plan, Some(key)); + } + for (old_key, new_key) in &plan.replacements { + if old_key == new_key { + continue; + } + let replacement = resolved.get(&old_key.name)?; + let old_snapshot = original_snapshots.get(old_key)?; + let dependencies = validate_dependencies( + effective_dependencies(&replacement.manifest)?, + old_snapshot.dependencies.as_ref(), + original_snapshots, + opts.lockfile.packages.as_ref()?, + plan, + new_key, + )?; + let optional_dependencies = validate_dependencies( + manifest_dependency_map(&replacement.manifest, "optionalDependencies")?, + old_snapshot.optional_dependencies.as_ref(), + original_snapshots, + opts.lockfile.packages.as_ref()?, + plan, + new_key, + )?; + let snapshot = + SnapshotEntry { dependencies, optional_dependencies, ..old_snapshot.clone() }; + if let Some(existing) = snapshots.get(new_key) + && existing != &snapshot + { + return None; + } + snapshots.insert(new_key.clone(), snapshot); + let metadata_key = new_key.without_peer(); + let metadata = package_metadata( + &replacement.manifest, + replacement.resolution.to_lockfile_form( + &old_key.name.to_string(), + &new_key.suffix.version().to_string(), + &pick_registry_for_package(opts.registries, &old_key.name.to_string(), None), + opts.lockfile_include_tarball_url, + ), + ); + if let Some(existing) = packages.get(&metadata_key) + && existing != &metadata + { + return None; + } + packages.insert(metadata_key, metadata); + } + updated.snapshots = Some(snapshots); + updated.packages = Some(packages); + updated.overrides = Some(opts.resolved_overrides.clone()); + prune_unreachable_packages(&mut updated); + Some(updated) +} + +fn rewrite_importer_dependencies( + dependencies: &mut Option, + plan: &RewritePlan, +) { + let Some(map) = dependencies else { return }; + map.retain(|alias, _| !should_remove_dependency(alias, None, &plan.overrides)); + for (alias, spec) in map.iter_mut() { + let Some(old_key) = spec.version.resolved_key(alias) else { continue }; + let Some(new_key) = plan.replacements.get(&old_key) else { continue }; + spec.version = ImporterDepVersion::Regular(new_key.suffix.clone()); + } + if map.is_empty() { + *dependencies = None; + } +} + +fn rewrite_snapshot_dependencies( + dependencies: &mut Option>, + plan: &RewritePlan, + parent_key: Option<&PackageKey>, +) { + let Some(map) = dependencies else { return }; + rewrite_snapshot_dependency_map(map, plan, parent_key); + if map.is_empty() { + *dependencies = None; + } +} + +fn rewrite_snapshot_dependency_map( + dependencies: &mut HashMap, + plan: &RewritePlan, + parent_key: Option<&PackageKey>, +) { + dependencies.retain(|alias, _| !should_remove_dependency(alias, parent_key, &plan.overrides)); + for (alias, dep_ref) in dependencies { + let Some(old_key) = dep_ref.resolve(alias) else { continue }; + let Some(new_key) = plan.replacements.get(&old_key) else { continue }; + *dep_ref = SnapshotDepRef::Plain(new_key.suffix.clone()); + } +} + +fn should_remove_dependency( + alias: &PkgName, + parent_key: Option<&PackageKey>, + overrides: &[FastOverride], +) -> bool { + overrides.iter().any(|override_entry| { + if override_entry.new_version.is_some() || override_entry.name != *alias { + return false; + } + let Some(parent) = override_entry.parent.as_ref() else { return true }; + let Some(parent_key) = parent_key else { return false }; + if parent_key.name.to_string() != parent.name { + return false; + } + match parent.bare_specifier.as_deref() { + None => true, + Some(range) => parent_key.suffix.version_semver().is_some_and(|version| { + Range::parse(range).is_ok_and(|range| range.satisfies(version)) + }), + } + }) +} + +fn prune_unreachable_packages(lockfile: &mut Lockfile) { + let reachable = { + let Some(snapshots) = lockfile.snapshots.as_ref() else { return }; + let mut reachable = HashSet::new(); + let mut queue = VecDeque::new(); + for importer in lockfile.importers.values() { + for dependencies in [ + importer.dependencies.as_ref(), + importer.dev_dependencies.as_ref(), + importer.optional_dependencies.as_ref(), + ] + .into_iter() + .flatten() + { + for (alias, spec) in dependencies { + if let Some(key) = spec.version.resolved_key(alias) { + queue.push_back(key); + } + } + } + } + while let Some(key) = queue.pop_front() { + if !reachable.insert(key.clone()) { + continue; + } + let Some(snapshot) = snapshots.get(&key) else { continue }; + for dependencies in + [snapshot.dependencies.as_ref(), snapshot.optional_dependencies.as_ref()] + .into_iter() + .flatten() + { + for (alias, dep_ref) in dependencies { + if let Some(key) = dep_ref.resolve(alias) { + queue.push_back(key); + } + } + } + } + reachable + }; + let reachable_metadata: HashSet<_> = + reachable.iter().map(PkgNameVerPeer::without_peer).collect(); + if let Some(snapshots) = lockfile.snapshots.as_mut() { + snapshots.retain(|key, _| reachable.contains(key)); + if snapshots.is_empty() { + lockfile.snapshots = None; + } + } + if let Some(packages) = lockfile.packages.as_mut() { + packages.retain(|key, _| reachable_metadata.contains(key)); + if packages.is_empty() { + lockfile.packages = None; + } + } +} + +fn validate_dependencies( + manifest_dependencies: HashMap, + locked_dependencies: Option<&HashMap>, + snapshots: &HashMap, + packages: &HashMap, + plan: &RewritePlan, + parent_key: &PackageKey, +) -> Option>> { + let locked_dependencies = locked_dependencies.cloned().unwrap_or_default(); + for name in locked_dependencies.keys() { + if !manifest_dependencies.contains_key(name) && plan.peer_names.contains(name) { + return None; + } + } + let mut rewritten = HashMap::new(); + for (name, range) in manifest_dependencies { + if should_remove_dependency(&name, Some(parent_key), &plan.overrides) { + continue; + } + let range = Range::parse(&range).ok()?; + let dep_ref = match locked_dependencies.get(&name) { + Some(dep_ref) => { + let mut dep_ref = dep_ref.clone(); + if let Some(old_key) = dep_ref.resolve(&name) + && let Some(new_key) = plan.replacements.get(&old_key) + { + dep_ref = SnapshotDepRef::Plain(new_key.suffix.clone()); + } + dep_ref + } + None => find_reusable_dependency(&name, &range, snapshots, packages, plan)?, + }; + let key = dep_ref.resolve(&name)?; + if !range.satisfies(key.suffix.version_semver()?) { + return None; + } + rewritten.insert(name, dep_ref); + } + Some((!rewritten.is_empty()).then_some(rewritten)) +} + +fn find_reusable_dependency( + name: &PkgName, + range: &Range, + snapshots: &HashMap, + packages: &HashMap, + plan: &RewritePlan, +) -> Option { + if plan.peer_names.contains(name) || plan.overrides.iter().any(|entry| entry.name == *name) { + return None; + } + let mut candidates = snapshots.iter().filter(|(key, snapshot)| { + if key.name != *name + || !key.suffix.peer().is_empty() + || key.suffix.prefix() != Prefix::None + || !key.suffix.version_semver().is_some_and(|version| range.satisfies(version)) + || snapshot.optional + || snapshot.patched == Some(true) + || snapshot.id.is_some() + || snapshot.transitive_peer_dependencies.is_some() + { + return false; + } + packages.get(&key.without_peer()).is_some_and(|metadata| { + metadata.peer_dependencies.is_none() + && metadata.peer_dependencies_meta.is_none() + && (matches!(metadata.resolution, LockfileResolution::Registry(_)) + || matches!( + metadata.resolution, + LockfileResolution::Tarball(ref tarball) + if tarball.integrity.is_some() && tarball.git_hosted != Some(true), + )) + }) + }); + let (key, _) = candidates.next()?; + candidates.next().is_none().then(|| SnapshotDepRef::Plain(key.suffix.clone())) +} + +fn effective_dependencies(manifest: &Value) -> Option> { + let optional = manifest_dependency_map(manifest, "optionalDependencies")?; + Some( + manifest_dependency_map(manifest, "dependencies")? + .into_iter() + .filter(|(name, _)| !optional.contains_key(name)) + .collect(), + ) +} + +fn manifest_dependency_map(manifest: &Value, key: &str) -> Option> { + let Some(value) = manifest.get(key) else { + return Some(HashMap::new()); + }; + let map = value.as_object()?; + map.iter() + .map(|(name, spec)| Some((PkgName::parse(name).ok()?, spec.as_str()?.to_string()))) + .collect() +} + +fn package_metadata(manifest: &Value, resolution: LockfileResolution) -> PackageMetadata { + PackageMetadata { + resolution, + version: None, + engines: string_map(manifest, "engines") + .map(|map| map.into_iter().filter(|(_, range)| range != "*").collect()) + .filter(|map: &HashMap<_, _>| !map.is_empty()), + cpu: string_list(manifest, "cpu"), + os: string_list(manifest, "os"), + libc: manifest.get("libc").and_then(|value| match value { + Value::String(value) => Some(StringOrList::String(value.clone())), + Value::Array(_) => string_list(manifest, "libc").map(StringOrList::List), + _ => None, + }), + deprecated: None, + has_bin: crate::dependencies_graph_to_lockfile::manifest_has_bin(Some(manifest)), + prepare: None, + bundled_dependencies: BundledDependencies::from_manifest(Some(manifest)), + peer_dependencies: None, + peer_dependencies_meta: None, + } +} + +fn string_map(manifest: &Value, key: &str) -> Option> { + let map = manifest.get(key)?.as_object()?; + Some( + map.iter() + .filter_map(|(name, value)| Some((name.clone(), value.as_str()?.to_string()))) + .collect(), + ) +} + +fn string_list(manifest: &Value, key: &str) -> Option> { + let values = manifest.get(key)?.as_array()?; + let values: Vec = + values.iter().filter_map(Value::as_str).map(ToString::to_string).collect(); + (!values.is_empty()).then_some(values) +} + +#[cfg(test)] +mod tests; diff --git a/pnpm/crates/package-manager/src/fast_update_overrides/tests.rs b/pnpm/crates/package-manager/src/fast_update_overrides/tests.rs new file mode 100644 index 0000000000..dd7d031658 --- /dev/null +++ b/pnpm/crates/package-manager/src/fast_update_overrides/tests.rs @@ -0,0 +1,496 @@ +use super::{FastOverrideOptions, try_fast_update_overrides}; +use indexmap::IndexMap; +use pacquet_config_parse_overrides::{PackageSelector, VersionOverride}; +use pacquet_lockfile::{Lockfile, LockfileResolution, PkgName, SnapshotEntry, TarballResolution}; +use pacquet_resolving_resolver_base::{ + LatestInfo, LatestQuery, PkgResolutionId, ResolveFuture, ResolveLatestFuture, ResolveOptions, + ResolveResult, Resolver, WantedDependency, +}; +use serde_json::json; +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +struct StubResolver { + calls: AtomicUsize, + manifest: serde_json::Value, +} + +impl Resolver for StubResolver { + fn resolve<'a>( + &'a self, + wanted_dependency: &'a WantedDependency, + _opts: &'a ResolveOptions, + ) -> ResolveFuture<'a> { + self.calls.fetch_add(1, Ordering::Relaxed); + let name = wanted_dependency.alias.clone().expect("alias"); + let version = wanted_dependency.bare_specifier.clone().expect("version"); + let manifest = Arc::new(self.manifest.clone()); + Box::pin(async move { + Ok(Some(ResolveResult { + id: PkgResolutionId::from(format!("{name}@{version}")), + name_ver: Some(format!("{name}@{version}").parse().expect("name and version")), + latest: Some(version), + published_at: None, + manifest: Some(manifest), + resolution: LockfileResolution::Tarball(TarballResolution { + tarball: "https://registry.npmjs.org/target/-/target-2.0.0.tgz".to_string(), + integrity: Some("sha512-dGFyZ2V0LTI=".parse().expect("integrity")), + git_hosted: None, + path: None, + }), + resolved_via: "npm-registry".to_string(), + normalized_bare_specifier: None, + alias: Some(name), + policy_violation: None, + })) + }) + } + + fn resolve_latest<'a>( + &'a self, + _query: &'a LatestQuery, + _opts: &'a ResolveOptions, + ) -> ResolveLatestFuture<'a> { + Box::pin(async { Ok(Some(LatestInfo { latest_manifest: None })) }) + } +} + +fn lockfile() -> Lockfile { + serde_json::from_value(json!({ + "lockfileVersion": "9.0", + "overrides": { + "target": "1.0.0" + }, + "importers": { + ".": { + "dependencies": { + "parent": { + "specifier": "1.0.0", + "version": "1.0.0" + } + } + } + }, + "packages": { + "parent@1.0.0": { + "resolution": { + "integrity": "sha512-parent" + } + }, + "target@1.0.0": { + "resolution": { + "integrity": "sha512-target-1" + } + }, + "child@1.1.0": { + "resolution": { + "integrity": "sha512-child" + } + } + }, + "snapshots": { + "parent@1.0.0": { + "dependencies": { + "target": "1.0.0" + } + }, + "target@1.0.0": { + "dependencies": { + "child": "1.1.0" + } + }, + "child@1.1.0": {} + } + })) + .expect("lockfile") +} + +fn parsed_override() -> Vec { + vec![VersionOverride { + selector: "target".to_string(), + parent_pkg: None, + target_pkg: PackageSelector { name: "target".to_string(), bare_specifier: None }, + new_bare_specifier: "2.0.0".to_string(), + converge: false, + }] +} + +async fn try_update( + lockfile: &Lockfile, + parsed_overrides: &[VersionOverride], + resolved_overrides: &IndexMap, + resolver: &dyn Resolver, +) -> Option { + let resolve_options = ResolveOptions::default(); + let registries = + HashMap::from([("default".to_string(), "https://registry.npmjs.org/".to_string())]); + try_fast_update_overrides(FastOverrideOptions { + lockfile, + parsed_overrides, + resolved_overrides, + resolver, + resolve_options: &resolve_options, + manifest_hook: None, + registries: ®istries, + lockfile_include_tarball_url: false, + }) + .await +} + +async fn update_with_manifest(manifest: serde_json::Value) -> (Option, usize) { + let lockfile = lockfile(); + let parsed = parsed_override(); + let overrides = IndexMap::from([("target".to_string(), "2.0.0".to_string())]); + let resolver = StubResolver { calls: AtomicUsize::new(0), manifest }; + let result = try_update(&lockfile, &parsed, &overrides, &resolver).await; + (result, resolver.calls.load(Ordering::Relaxed)) +} + +#[tokio::test] +async fn rewrites_an_exact_override_when_locked_children_satisfy_the_new_manifest() { + let (updated, calls) = update_with_manifest(json!({ + "name": "target", + "version": "2.0.0", + "dependencies": { + "child": "^1.0.0" + } + })) + .await; + let updated = updated.expect("fast override update"); + let target = PkgName::parse("target").expect("package name"); + let parent_key = "parent@1.0.0".parse().expect("parent key"); + let parent = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&parent_key)) + .expect("parent snapshot"); + + assert_eq!(calls, 1); + assert_eq!( + parent + .dependencies + .as_ref() + .and_then(|dependencies| dependencies.get(&target)) + .map(ToString::to_string) + .as_deref(), + Some("2.0.0"), + ); + assert!( + updated + .snapshots + .as_ref() + .is_some_and(|snapshots| snapshots.contains_key(&"target@2.0.0".parse().unwrap())), + ); +} + +#[tokio::test] +async fn falls_back_when_a_locked_child_does_not_satisfy_the_new_manifest() { + let (updated, calls) = update_with_manifest(json!({ + "name": "target", + "version": "2.0.0", + "dependencies": { + "child": "^2.0.0" + } + })) + .await; + + assert_eq!(calls, 1); + assert!(updated.is_none()); +} + +#[tokio::test] +async fn falls_back_when_registry_metadata_has_invalid_dependency_fields() { + for manifest in [ + json!({ + "name": "target", + "version": "2.0.0", + "peerDependencies": "" + }), + json!({ + "name": "target", + "version": "2.0.0", + "peerDependenciesMeta": [] + }), + json!({ + "name": "target", + "version": "2.0.0", + "engines": "node" + }), + ] { + let (updated, calls) = update_with_manifest(manifest).await; + assert_eq!(calls, 1); + assert!(updated.is_none()); + } +} + +#[tokio::test] +async fn drops_obsolete_dependency_edges_from_a_replacement() { + let (updated, calls) = update_with_manifest(json!({ + "name": "target", + "version": "2.0.0" + })) + .await; + let updated = updated.expect("fast override update"); + let target = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&"target@2.0.0".parse().unwrap())) + .expect("target snapshot"); + + assert_eq!(calls, 1); + assert!(target.dependencies.is_none()); + assert!( + updated + .snapshots + .as_ref() + .is_some_and(|snapshots| { !snapshots.contains_key(&"child@1.1.0".parse().unwrap()) }), + ); +} + +#[tokio::test] +async fn reuses_a_unique_compatible_locked_dependency_added_by_a_replacement() { + let mut lockfile = lockfile(); + let importer = lockfile.importers.get_mut(".").expect("root importer"); + importer.dependencies.as_mut().expect("dependencies").insert( + PkgName::parse("added").unwrap(), + serde_json::from_value(json!({ + "specifier": "1.0.0", + "version": "1.0.0" + })) + .unwrap(), + ); + lockfile.packages.as_mut().expect("packages").insert( + "added@1.0.0".parse().unwrap(), + serde_json::from_value(json!({ + "resolution": { + "integrity": "sha512-added" + } + })) + .unwrap(), + ); + lockfile + .snapshots + .as_mut() + .expect("snapshots") + .insert("added@1.0.0".parse().unwrap(), SnapshotEntry::default()); + let parsed = parsed_override(); + let overrides = IndexMap::from([("target".to_string(), "2.0.0".to_string())]); + let resolver = StubResolver { + calls: AtomicUsize::new(0), + manifest: json!({ + "name": "target", + "version": "2.0.0", + "dependencies": { + "child": "^1.0.0", + "added": "^1.0.0" + } + }), + }; + + let updated = + try_update(&lockfile, &parsed, &overrides, &resolver).await.expect("fast override update"); + let target = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&"target@2.0.0".parse().unwrap())) + .expect("target snapshot"); + + assert_eq!(resolver.calls.load(Ordering::Relaxed), 1); + assert_eq!( + target + .dependencies + .as_ref() + .and_then(|dependencies| dependencies.get(&PkgName::parse("added").unwrap())) + .map(ToString::to_string) + .as_deref(), + Some("1.0.0"), + ); +} + +async fn remove_target(lockfile: &Lockfile) -> (Option, usize) { + let parsed = vec![VersionOverride { + selector: "target".to_string(), + parent_pkg: None, + target_pkg: PackageSelector { name: "target".to_string(), bare_specifier: None }, + new_bare_specifier: "-".to_string(), + converge: false, + }]; + let overrides = IndexMap::from([("target".to_string(), "-".to_string())]); + let resolver = StubResolver { + calls: AtomicUsize::new(0), + manifest: json!({ + "name": "target", + "version": "2.0.0" + }), + }; + let result = try_update(lockfile, &parsed, &overrides, &resolver).await; + (result, resolver.calls.load(Ordering::Relaxed)) +} + +#[tokio::test] +async fn removes_a_dependency_and_its_unreachable_subtree_without_resolving() { + let (updated, calls) = remove_target(&lockfile()).await; + let updated = updated.expect("fast dependency removal"); + let parent_key = "parent@1.0.0".parse().expect("parent key"); + let parent = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&parent_key)) + .expect("parent snapshot"); + + assert_eq!(calls, 0); + assert!(parent.dependencies.is_none()); + assert!(updated.snapshots.as_ref().is_some_and(|snapshots| { + !snapshots.contains_key(&"target@1.0.0".parse().unwrap()) + && !snapshots.contains_key(&"child@1.1.0".parse().unwrap()) + })); + assert!(updated.packages.as_ref().is_some_and(|packages| { + !packages.contains_key(&"target@1.0.0".parse().unwrap()) + && !packages.contains_key(&"child@1.1.0".parse().unwrap()) + })); +} + +#[tokio::test] +async fn falls_back_when_the_removed_dependency_is_used_as_a_peer() { + let mut lockfile = lockfile(); + lockfile + .packages + .as_mut() + .and_then(|packages| packages.get_mut(&"parent@1.0.0".parse().unwrap())) + .expect("parent metadata") + .peer_dependencies = Some(HashMap::from([("target".to_string(), "^1.0.0".to_string())])); + + let (updated, calls) = remove_target(&lockfile).await; + + assert_eq!(calls, 0); + assert!(updated.is_none()); +} + +#[tokio::test] +async fn removes_a_dependency_only_from_matching_parent_snapshots() { + let mut lockfile = lockfile(); + lockfile.overrides = None; + let parsed = vec![VersionOverride { + selector: "parent@^1>target".to_string(), + parent_pkg: Some(PackageSelector { + name: "parent".to_string(), + bare_specifier: Some("^1".to_string()), + }), + target_pkg: PackageSelector { name: "target".to_string(), bare_specifier: None }, + new_bare_specifier: "-".to_string(), + converge: false, + }]; + let overrides = IndexMap::from([("parent@^1>target".to_string(), "-".to_string())]); + let resolver = StubResolver { + calls: AtomicUsize::new(0), + manifest: json!({ + "name": "target", + "version": "2.0.0" + }), + }; + + let updated = try_update(&lockfile, &parsed, &overrides, &resolver) + .await + .expect("fast dependency removal"); + let parent = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&"parent@1.0.0".parse().unwrap())) + .expect("parent snapshot"); + + assert_eq!(resolver.calls.load(Ordering::Relaxed), 0); + assert!(parent.dependencies.is_none()); +} + +#[tokio::test] +async fn applies_exact_replacements_and_dependency_removals_together() { + let mut lockfile = lockfile(); + lockfile.packages.as_mut().expect("packages").insert( + "obsolete@1.0.0".parse().unwrap(), + serde_json::from_value(json!({ + "resolution": { + "integrity": "sha512-obsolete" + } + })) + .unwrap(), + ); + lockfile + .snapshots + .as_mut() + .expect("snapshots") + .insert("obsolete@1.0.0".parse().unwrap(), SnapshotEntry::default()); + lockfile + .snapshots + .as_mut() + .and_then(|snapshots| snapshots.get_mut(&"parent@1.0.0".parse().unwrap())) + .and_then(|parent| parent.dependencies.as_mut()) + .expect("parent dependencies") + .insert(PkgName::parse("obsolete").unwrap(), "1.0.0".parse().unwrap()); + let parsed = vec![ + parsed_override().remove(0), + VersionOverride { + selector: "obsolete".to_string(), + parent_pkg: None, + target_pkg: PackageSelector { name: "obsolete".to_string(), bare_specifier: None }, + new_bare_specifier: "-".to_string(), + converge: false, + }, + ]; + let overrides = IndexMap::from([ + ("target".to_string(), "2.0.0".to_string()), + ("obsolete".to_string(), "-".to_string()), + ]); + let resolver = StubResolver { + calls: AtomicUsize::new(0), + manifest: json!({ + "name": "target", + "version": "2.0.0", + "dependencies": { + "child": "^1.0.0", + "obsolete": "^1.0.0" + } + }), + }; + + let updated = try_update(&lockfile, &parsed, &overrides, &resolver) + .await + .expect("fast mixed override update"); + let parent = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&"parent@1.0.0".parse().unwrap())) + .expect("parent snapshot"); + + assert_eq!(resolver.calls.load(Ordering::Relaxed), 1); + assert_eq!( + parent + .dependencies + .as_ref() + .and_then(|dependencies| dependencies.get(&PkgName::parse("target").unwrap())) + .map(ToString::to_string) + .as_deref(), + Some("2.0.0"), + ); + assert!(parent.dependencies.as_ref().is_none_or(|dependencies| { + !dependencies.contains_key(&PkgName::parse("obsolete").unwrap()) + })); + let replacement = updated + .snapshots + .as_ref() + .and_then(|snapshots| snapshots.get(&"target@2.0.0".parse().unwrap())) + .expect("replacement snapshot"); + assert!(replacement.dependencies.as_ref().is_none_or(|dependencies| { + !dependencies.contains_key(&PkgName::parse("obsolete").unwrap()) + })); + assert!( + updated + .snapshots + .as_ref() + .is_some_and(|snapshots| !snapshots.contains_key(&"obsolete@1.0.0".parse().unwrap())), + ); +} diff --git a/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs b/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs index 79db76b2a7..5521a1920c 100644 --- a/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs +++ b/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs @@ -5,7 +5,9 @@ use crate::{ LinkVirtualStoreBins, LinkVirtualStoreBinsError, PrefetchContext, PrefetchingResolver, SkippedSnapshots, SymlinkDirectDependencies, SymlinkDirectDependenciesError, VersionPolicyError, VersionsOverrider, VirtualStoreLayout, dependencies_graph_to_lockfile, - link_root_component_members, store_init::init_store_dir_best_effort, + fast_update_overrides::{FastOverrideOptions, try_fast_update_overrides}, + link_root_component_members, + store_init::init_store_dir_best_effort, }; use dashmap::DashMap; use derive_more::{Display, Error}; @@ -697,6 +699,7 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { let minimum_release_age_exclude_override = resolution_observer .as_ref() .and_then(|observer| observer.minimum_release_age_exclude_override()); + let can_fast_update_overrides = resolution_observer.is_none(); let is_hoisted = matches!(node_linker, NodeLinker::Hoisted); let extra_node_paths = crate::shim_extra_node_paths(config, node_linker); let filtered_isolated = @@ -1379,17 +1382,63 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { update_reuse_scopes_by_importer.clear(); } + let reusable_settings_lockfile = wanted_lockfile + .filter(|lockfile| lockfile.package_extensions_checksum == package_extensions_checksum); + let override_settings_match = reusable_settings_lockfile.is_some_and(|lockfile| { + overrides_match(lockfile.overrides.as_ref(), resolved_overrides.as_ref()) + }); + let fast_override_seed = if let (Some(lockfile), Some(parsed), Some(resolved)) = + (reusable_settings_lockfile, parsed_overrides.as_deref(), resolved_overrides.as_ref()) + && !override_settings_match + && pnpmfile_hook.is_none() + && custom_resolvers_raw.is_empty() + && patched_dependencies.is_none() + && can_fast_update_overrides + { + let resolve_options = ResolveOptions { + preferred_versions: Arc::clone(&preferred_versions_seed), + default_tag: Some("latest".to_string()), + published_by, + published_by_exclude: published_by_exclude.clone(), + trust_policy, + trust_policy_exclude: trust_policy_exclude.clone(), + trust_policy_ignore_after: config.trust_policy_ignore_after, + package_version_guard: package_version_guard.clone(), + project_dir: lockfile_dir.to_path_buf(), + lockfile_dir: lockfile_dir.to_path_buf(), + workspace_packages: workspace_packages.clone(), + block_exotic_subdeps: config.block_exotic_subdeps, + always_try_workspace_packages: config.link_workspace_packages + != LinkWorkspacePackages::Off, + inject_workspace_packages: config.inject_workspace_packages, + prefer_workspace_packages: config.prefer_workspace_packages, + update_checksums, + ..ResolveOptions::default() + }; + try_fast_update_overrides(FastOverrideOptions { + lockfile, + parsed_overrides: parsed, + resolved_overrides: resolved, + resolver: &*npm_resolver, + resolve_options: &resolve_options, + manifest_hook: manifest_hook.as_ref(), + registries: ®istries, + lockfile_include_tarball_url: config.lockfile_include_tarball_url, + }) + .await + } else { + None + }; + // Hand the resolver the prior lockfile so it can reuse // already-resolved subtrees instead of re-resolving from the // registry (see pnpm/plans/LOCKFILE_RESOLUTION_REUSE.md). - // Withhold it when packageExtensions or overrides drifted: - // both settings rewrite package dependency sets, so the - // recorded subtree is stale. pnpm likewise invalidates the - // lockfile on these settings changes. - let lockfile_reuse_seed = wanted_lockfile.filter(|lockfile| { - lockfile.package_extensions_checksum == package_extensions_checksum - && overrides_match(lockfile.overrides.as_ref(), resolved_overrides.as_ref()) - }); + // Exact generic registry overrides may instead use a + // dependency-shape-verified rewritten seed. Every unsupported + // override shape falls back to withholding the seed. + let lockfile_reuse_seed = fast_override_seed + .as_ref() + .or_else(|| override_settings_match.then_some(reusable_settings_lockfile).flatten()); // Reused subtrees never stream their manifests through the // versions overrider, so only a resolution with no reuse at all // collects the complete declared-range set the convergence @@ -2106,8 +2155,8 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { config, packages: materialization_lockfile.packages.as_ref(), snapshots: materialization_lockfile.snapshots.as_ref(), - current_snapshots: None, - current_packages: None, + current_snapshots: current_lockfile.and_then(|lockfile| lockfile.snapshots.as_ref()), + current_packages: current_lockfile.and_then(|lockfile| lockfile.packages.as_ref()), layout: &layout, logged_methods, requester, diff --git a/pnpm/crates/package-manager/src/lib.rs b/pnpm/crates/package-manager/src/lib.rs index 7554658863..4727cbb985 100644 --- a/pnpm/crates/package-manager/src/lib.rs +++ b/pnpm/crates/package-manager/src/lib.rs @@ -13,6 +13,7 @@ mod create_virtual_store; mod current_lockfile; mod dependencies_graph_to_lockfile; mod deps_graph; +mod fast_update_overrides; mod graph_sequencer; mod hoist; mod hoisted_dep_graph; diff --git a/pnpm/plans/TEST_PORTING.md b/pnpm/plans/TEST_PORTING.md index e67937a534..31c22656b0 100644 --- a/pnpm/plans/TEST_PORTING.md +++ b/pnpm/plans/TEST_PORTING.md @@ -882,6 +882,12 @@ Rust port notes: - The cache cross-stack contract is content-divergent on hash format only — pacquet writes sha256-**hex** where pnpm writes sha256-**base64** (object-hash's default). Each stack reads its own records out of the shared JSONL; cross-stack hits aren't expected and aren't tested. - The end-to-end CLI test uses a 100-year `minimumReleaseAge` to sidestep the mocked registry's real-world `time` field. A finer-grained fixture with controlled `time` values lives in the unit tests (`fetch_full_metadata_cached::tests`, `create_npm_resolution_verifier::tests`). +## Exact Override Lockfile Reuse + +- [x] `TypeScript repo: pnpm11/installing/deps-installer/test/install/overrides.ts` `adding an exact override reuses the lockfile when the new package has the same dependencies` and `an exact override update reuses the lockfile when the new package has the same dependencies` — the rewrite core is pinned by `fast_update_overrides::tests::rewrites_an_exact_override_when_locked_children_satisfy_the_new_manifest`, including the single replacement-manifest resolve; `pacquet-cli::lockfile_resolution_reuse::exact_override_update_reuses_the_locked_children` covers wanted/current lockfile and materialization integration. +- [x] `TypeScript repo: pnpm11/installing/deps-installer/test/install/overrides.ts` `an exact override update falls back to resolution when the package dependencies changed` — `fast_update_overrides::tests::falls_back_when_a_locked_child_does_not_satisfy_the_new_manifest`. +- [x] `TypeScript repo: pnpm11/installing/deps-installer/test/install/overrides.ts` `a dependency removal override prunes the locked subtree without resolution` — the rewrite and peer-safety gates are pinned by `fast_update_overrides::tests::removes_a_dependency_and_its_unreachable_subtree_without_resolving` and `falls_back_when_the_removed_dependency_is_used_as_a_peer`; `pacquet-cli::lockfile_resolution_reuse::dependency_removal_override_prunes_the_locked_subtree_without_resolving` verifies the wanted/current lockfiles and succeeds with the registry unavailable. + ## `optimisticRepeatInstall` + `checkDepsStatus` Pre-Install Shortcut Tracks pnpm/pnpm#11940. Pacquet's port (`pacquet-package-manager::optimistic_repeat_install`) covers the mtime-vs-`lastValidatedTimestamp` branch of upstream's `checkDepsStatus`. Ported tests live in `optimistic_repeat_install::tests` and the install-level `optimistic_repeat_install_skips_entire_pipeline_when_state_is_fresh` end-to-end. diff --git a/pnpm11/installing/deps-installer/src/install/index.ts b/pnpm11/installing/deps-installer/src/install/index.ts index 341fbe8602..c7fc098bb8 100644 --- a/pnpm11/installing/deps-installer/src/install/index.ts +++ b/pnpm11/installing/deps-installer/src/install/index.ts @@ -9,6 +9,7 @@ import { parseCatalogProtocol } from '@pnpm/catalogs.protocol-parser' import { type CatalogResultMatcher, matchCatalogResolveResult, resolveFromCatalog } from '@pnpm/catalogs.resolver' import type { Catalogs } from '@pnpm/catalogs.types' import { parseOverrides } from '@pnpm/config.parse-overrides' +import { createPackageVersionPolicyOrThrow, getPublishedByPolicy } from '@pnpm/config.version-policy' import { LAYOUT_VERSION, LOCKFILE_MAJOR_VERSION, @@ -106,6 +107,7 @@ import { } from './extendInstallOptions.js' import { linkPackages } from './link.js' import { reportPeerDependencyIssues } from './reportPeerDependencyIssues.js' +import { tryFastUpdateOverrides } from './tryFastUpdateOverrides.js' import { validateModules } from './validateModules.js' import { verifyLockfileResolutions } from './verifyLockfileResolutions.js' import { warnOnStaleConvergenceOverrides } from './warnOnStaleConvergenceOverrides.js' @@ -662,29 +664,107 @@ export async function mutateModules ( const patchGroups = patchGroupInput ? groupPatchedDependencies(patchGroupInput) : undefined const frozenLockfile = opts.frozenLockfile || opts.frozenLockfileIfExists && ctx.existsNonEmptyWantedLockfile - let outdatedLockfileSettings = false + let outdatedLockfileSettingName = null as ReturnType const overridesMap = createOverridesMapFromParsed(opts.parsedOverrides) + const lockfileSettings = { + autoInstallPeers: opts.autoInstallPeers, + catalogs: opts.catalogs, + dedupePeers: opts.dedupePeers || undefined, + injectWorkspacePackages: opts.injectWorkspacePackages, + excludeLinksFromLockfile: opts.excludeLinksFromLockfile, + peersSuffixMaxLength: opts.peersSuffixMaxLength, + ignoredOptionalDependencies: opts.ignoredOptionalDependencies?.sort(), + packageExtensionsChecksum, + patchedDependencies, + pnpmfileChecksum, + } if (!opts.ignorePackageManifest) { - const outdatedLockfileSettingName = getOutdatedLockfileSetting(ctx.wantedLockfile, { - autoInstallPeers: opts.autoInstallPeers, - catalogs: opts.catalogs, - dedupePeers: opts.dedupePeers || undefined, - injectWorkspacePackages: opts.injectWorkspacePackages, - excludeLinksFromLockfile: opts.excludeLinksFromLockfile, - peersSuffixMaxLength: opts.peersSuffixMaxLength, + outdatedLockfileSettingName = getOutdatedLockfileSetting(ctx.wantedLockfile, { + ...lockfileSettings, overrides: overridesMap, - ignoredOptionalDependencies: opts.ignoredOptionalDependencies?.sort(), - packageExtensionsChecksum, - patchedDependencies, - pnpmfileChecksum, }) - outdatedLockfileSettings = outdatedLockfileSettingName != null - if (frozenLockfile && outdatedLockfileSettings) { + if (frozenLockfile && outdatedLockfileSettingName != null) { throw new LockfileConfigMismatchError(outdatedLockfileSettingName!) } } const _isWantedDepBareSpecifierSame = isWantedDepBareSpecifierSame.bind(null, ctx.wantedLockfile.catalogs, opts.catalogs) const upToDateLockfileMajorVersion = ctx.wantedLockfile.lockfileVersion.toString().startsWith(`${LOCKFILE_MAJOR_VERSION}.`) + let didFastUpdateOverrides = false + const canTryFastUpdateOverrides = + outdatedLockfileSettingName === 'overrides' && + installsOnly && + !isCheckOnlyInstall(opts) && + opts.preferFrozenLockfile && + opts.useLockfile && + opts.saveLockfile && + opts.runPacquet == null && + !opts.fixLockfile && + !opts.dedupe && + !opts.updateChecksums && + !opts.force && + !opts.forceFullResolution && + !forceResolutionFromHook && + !opts.hooks.readPackage?.length && + !opts.hooks.preResolution?.length && + !opts.hooks.afterAllResolved?.length && + opts.hooks.customResolvers == null && + !ctx.lockfileHadConflicts && + ctx.wantedLockfile.lockfileVersion === LOCKFILE_VERSION && + !isEmptyLockfile(ctx.wantedLockfile) && + (!opts.pruneLockfileImporters || Object.keys(ctx.wantedLockfile.importers).length === Object.keys(ctx.projects).length) && + ctx.wantedLockfile.time == null + if (canTryFastUpdateOverrides) { + await verifyLockfilePromise + const { publishedBy, publishedByExclude } = getPublishedByPolicy(opts) + if ( + // The helper reports only the first mismatch, so checking with the + // lockfile's overrides proves overrides were the only stale setting. + getOutdatedLockfileSetting(ctx.wantedLockfile, { + ...lockfileSettings, + overrides: ctx.wantedLockfile.overrides, + }) == null && + await allProjectsAreUpToDate(Object.values(ctx.projects), { + catalogs: opts.catalogs, + autoInstallPeers: opts.autoInstallPeers, + excludeLinksFromLockfile: opts.excludeLinksFromLockfile, + linkWorkspacePackages: opts.linkWorkspacePackagesDepth >= 0, + wantedLockfile: ctx.wantedLockfile, + workspacePackages: ctx.workspacePackages, + lockfileDir: opts.lockfileDir, + }) && + await tryFastUpdateOverrides(ctx.wantedLockfile, { + lockfileDir: opts.lockfileDir, + lockfileIncludeTarballUrl: opts.lockfileIncludeTarballUrl, + overrides: overridesMap, + parsedOverrides: opts.parsedOverrides, + readPackageHook: opts.readPackageHook, + registries: ctx.registries, + requestPackage: opts.storeController.requestPackage, + publishedBy, + publishedByExclude, + trustPolicy: opts.trustPolicy, + trustPolicyExclude: opts.trustPolicyExclude + ? createPackageVersionPolicyOrThrow(opts.trustPolicyExclude, 'trustPolicyExclude') + : undefined, + trustPolicyIgnoreAfter: opts.trustPolicyIgnoreAfter, + isLockfileUpToDate: (lockfile) => allProjectsAreUpToDate(Object.values(ctx.projects), { + catalogs: opts.catalogs, + autoInstallPeers: opts.autoInstallPeers, + excludeLinksFromLockfile: opts.excludeLinksFromLockfile, + linkWorkspacePackages: opts.linkWorkspacePackagesDepth >= 0, + wantedLockfile: lockfile, + workspacePackages: ctx.workspacePackages, + lockfileDir: opts.lockfileDir, + }), + verifyLockfile: (lockfile) => verifyLockfileResolutions(lockfile, []), + }) + ) { + outdatedLockfileSettingName = null + ctx.wantedLockfileIsModified = true + didFastUpdateOverrides = true + } + } + const outdatedLockfileSettings = outdatedLockfileSettingName != null let needsFullResolution = outdatedLockfileSettings || opts.fixLockfile || opts.updateChecksums || @@ -715,6 +795,7 @@ export async function mutateModules ( } const frozenInstallResult = await tryFrozenInstall({ + didFastUpdateOverrides, frozenLockfile, needsFullResolution, patchGroups, @@ -985,11 +1066,13 @@ export async function mutateModules ( * not change recorded dependency resolutions. */ async function tryFrozenInstall ({ + didFastUpdateOverrides, frozenLockfile, needsFullResolution, patchGroups, upToDateLockfileMajorVersion, }: { + didFastUpdateOverrides: boolean frozenLockfile: boolean needsFullResolution: boolean patchGroups?: PatchGroupRecord @@ -1140,6 +1223,7 @@ Note that in CI environments, this setting is enabled by default.`, allProjects: ctx.projects, prunedAt: ctx.modulesFile?.prunedAt, pruneVirtualStore, + relinkChangedDependenciesOnly: didFastUpdateOverrides, wantedLockfile: maybeOpts.ignorePackageManifest ? undefined : ctx.wantedLockfile, useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified, verifyLockfile, diff --git a/pnpm11/installing/deps-installer/src/install/link.ts b/pnpm11/installing/deps-installer/src/install/link.ts index 3582af35f0..82bbc8e647 100644 --- a/pnpm11/installing/deps-installer/src/install/link.ts +++ b/pnpm11/installing/deps-installer/src/install/link.ts @@ -9,16 +9,15 @@ import { import { calcDepState, type DepsStateCache, findRuntimeNodeVersion } from '@pnpm/deps.graph-hasher' import { readModulesDir } from '@pnpm/fs.read-modules-dir' import { symlinkDependency } from '@pnpm/fs.symlink-dependency' -import { - type DependenciesGraph, - type DependenciesGraphNode, - isValidDependencyAlias, - type LinkedDependency, +import type { + DependenciesGraph, + DependenciesGraphNode, + LinkedDependency, } from '@pnpm/installing.deps-resolver' import type { InstallationResultStats } from '@pnpm/installing.deps-restorer' import { linkDirectDeps } from '@pnpm/installing.linking.direct-dep-linker' import { hoist, type HoistedWorkspaceProject } from '@pnpm/installing.linking.hoist' -import { prune } from '@pnpm/installing.linking.modules-cleaner' +import { prune, removeObsoleteDependency } from '@pnpm/installing.linking.modules-cleaner' import type { IncludedDependencies } from '@pnpm/installing.modules-yaml' import { filterLockfileByImporters, @@ -35,7 +34,6 @@ import type { SupportedArchitectures, } from '@pnpm/types' import { symlinkAllModules } from '@pnpm/worker' -import { rimraf } from '@zkochan/rimraf' import pLimit from 'p-limit' import { pathExists } from 'path-exists' import { difference, equals, isEmpty, pick, pickBy, props } from 'ramda' @@ -571,7 +569,7 @@ async function linkAllModules ( optional: boolean } ): Promise { - await Promise.all(depNodes.flatMap((depNode) => (depNode.removedAliases ?? []).map(async (alias) => limitModulesDirReads(async () => removeObsoleteChild(depNode.modules, alias))))) + await Promise.all(depNodes.flatMap((depNode) => (depNode.removedAliases ?? []).map(async (alias) => limitModulesDirReads(async () => removeObsoleteDependency(depNode.modules, alias))))) await symlinkAllModules({ deps: depNodes.map((depNode) => { return { @@ -639,15 +637,6 @@ async function getActualChildrenDiff ( return { actualChildrenChanged, removedAliases } } -async function removeObsoleteChild (modulesDir: string, alias: string): Promise { - // Guard against an alias that would escape the modules directory (e.g. `../../x`). - if (!isValidDependencyAlias(alias)) return - await rimraf(path.join(modulesDir, alias)) - if (alias[0] === '@') { - await fs.rmdir(path.join(modulesDir, alias.split('/')[0])).catch(() => {}) - } -} - function getChildrenPaths ( depNode: ModulesLinkJob, depGraph: DependenciesGraph, diff --git a/pnpm11/installing/deps-installer/src/install/tryFastUpdateOverrides.ts b/pnpm11/installing/deps-installer/src/install/tryFastUpdateOverrides.ts new file mode 100644 index 0000000000..6a6ab0440e --- /dev/null +++ b/pnpm11/installing/deps-installer/src/install/tryFastUpdateOverrides.ts @@ -0,0 +1,590 @@ +import type { VersionOverride } from '@pnpm/config.parse-overrides' +import * as dp from '@pnpm/deps.path' +import type { + LockfileObject, + PackageSnapshot, + ResolvedDependencies, +} from '@pnpm/lockfile.types' +import { toLockfileResolution } from '@pnpm/lockfile.utils' +import type { RequestPackageFunction } from '@pnpm/store.controller-types' +import type { + DepPath, + PackageManifest, + PackageVersionPolicy, + ReadPackageHook, + Registries, + TrustPolicy, +} from '@pnpm/types' +import { clone, equals } from 'ramda' +import semver from 'semver' + +interface FastOverride { + name: string + newVersion?: string + oldVersion?: string + parent?: { + name: string + bareSpecifier?: string + } +} + +interface RewriteContext { + changedNames: Set + peerNames: Set + removals: FastOverride[] + replacements: Map +} + +interface ResolverPolicyOptions { + publishedBy?: Date + publishedByExclude?: PackageVersionPolicy + trustPolicy?: TrustPolicy + trustPolicyExclude?: PackageVersionPolicy + trustPolicyIgnoreAfter?: number +} + +export async function tryFastUpdateOverrides ( + lockfile: LockfileObject, + opts: ResolverPolicyOptions & { + lockfileDir: string + lockfileIncludeTarballUrl?: boolean + overrides: Record + parsedOverrides: VersionOverride[] + isLockfileUpToDate: (lockfile: LockfileObject) => Promise + readPackageHook?: ReadPackageHook + registries: Registries + requestPackage: RequestPackageFunction + verifyLockfile?: (lockfile: LockfileObject) => Promise + } +): Promise { + const fastOverrides = getFastOverrides(lockfile.overrides ?? {}, opts.overrides, opts.parsedOverrides) + if (fastOverrides == null) return false + + const removals = fastOverrides.filter(({ newVersion }) => newVersion == null) + const peerNames = getPeerNames(lockfile) + if (removals.some(({ name }) => peerNames.has(name))) return false + + const replacements = collectReplacements(lockfile, fastOverrides) + if (replacements == null) return false + + const changedNames = new Set( + fastOverrides + .filter(({ newVersion }) => newVersion != null) + .map(({ name }) => name) + ) + const rewriteContext = { changedNames, peerNames, removals, replacements } + const manifests = await resolveNewManifests(fastOverrides, replacements, opts) + if (manifests == null) return false + + const packages = rewritePackages(lockfile.packages ?? {}, { + lockfileIncludeTarballUrl: opts.lockfileIncludeTarballUrl, + manifests, + registries: opts.registries, + rewriteContext, + }) + if (packages == null) return false + + const importers = Object.fromEntries( + Object.entries(lockfile.importers).map(([id, importer]) => [ + id, + { + ...importer, + dependencies: rewriteResolvedDependencies(importer.dependencies, rewriteContext), + devDependencies: rewriteResolvedDependencies(importer.devDependencies, rewriteContext), + optionalDependencies: rewriteResolvedDependencies(importer.optionalDependencies, rewriteContext), + }, + ]) + ) as LockfileObject['importers'] + + const updatedLockfile: LockfileObject = { + ...lockfile, + importers, + packages: pruneUnreachablePackages(importers, packages), + overrides: opts.overrides, + } + if (!await opts.isLockfileUpToDate(updatedLockfile)) return false + await opts.verifyLockfile?.(updatedLockfile) + lockfile.importers = updatedLockfile.importers + lockfile.packages = updatedLockfile.packages + lockfile.overrides = updatedLockfile.overrides + return true +} + +function getFastOverrides ( + oldOverrides: Record, + newOverrides: Record, + parsedOverrides: VersionOverride[] +): FastOverride[] | null { + if (Object.keys(oldOverrides).some((selector) => newOverrides[selector] == null)) return null + + const changedSelectors = Object.keys(newOverrides) + .filter((selector) => oldOverrides[selector] !== newOverrides[selector]) + if (changedSelectors.length === 0) return null + + const parsedBySelector = new Map(parsedOverrides.map((override) => [override.selector, override])) + const changedNames = new Set() + const result: FastOverride[] = [] + for (const selector of changedSelectors) { + const override = parsedBySelector.get(selector) + const newValue = newOverrides[selector] + const oldVersion = oldOverrides[selector] + const removesDependency = newValue === '-' + if ( + override == null || + override.targetPkg.bareSpecifier != null || + override.converge === true || + !removesDependency && ( + override.parentPkg != null || + semver.valid(newValue) == null || + oldVersion != null && semver.valid(oldVersion) == null + ) || + changedNames.has(override.targetPkg.name) || + parsedOverrides.some((candidate) => + candidate.selector !== selector && + candidate.targetPkg.name === override.targetPkg.name + ) + ) { + return null + } + changedNames.add(override.targetPkg.name) + result.push({ + name: override.targetPkg.name, + ...override.parentPkg == null + ? {} + : { + parent: { + name: override.parentPkg.name, + bareSpecifier: override.parentPkg.bareSpecifier, + }, + }, + ...removesDependency ? {} : { newVersion: newValue, oldVersion }, + }) + } + return result +} + +function collectReplacements ( + lockfile: LockfileObject, + overrides: FastOverride[] +): Map | null { + const overridesByName = new Map(overrides.map((override) => [override.name, override])) + const replacements = new Map() + for (const dependencies of allResolvedDependencyMaps(lockfile)) { + for (const [alias, reference] of Object.entries(dependencies)) { + const override = overridesByName.get(alias) + if (override?.newVersion == null) continue + const oldDepPath = dp.refToRelative(reference, alias) + if (oldDepPath == null) return null + const parsed = dp.parse(oldDepPath) + const snapshot = lockfile.packages?.[oldDepPath] + if ( + parsed.name !== alias || + parsed.version == null || + parsed.peerDepGraphHash != null || + parsed.patchHash != null || + override.oldVersion != null && parsed.version !== override.oldVersion || + snapshot == null || + snapshot.optional === true || + snapshot.peerDependencies != null || + snapshot.peerDependenciesMeta != null || + !('integrity' in snapshot.resolution) || + typeof snapshot.resolution.integrity !== 'string' || + 'type' in snapshot.resolution && snapshot.resolution.type != null + ) { + return null + } + const newDepPath = `${alias}@${override.newVersion}${parsed.peerDepGraphHash ?? ''}` as DepPath + const previousReplacement = replacements.get(oldDepPath) + if (previousReplacement != null && previousReplacement !== newDepPath) return null + replacements.set(oldDepPath, newDepPath) + } + } + for (const dependencies of allResolvedDependencyMaps(lockfile)) { + for (const [alias, reference] of Object.entries(dependencies)) { + const depPath = dp.refToRelative(reference, alias) + if (depPath != null && replacements.has(depPath) && !overridesByName.has(alias)) return null + } + } + return replacements +} + +function getPeerNames (lockfile: LockfileObject): Set { + const result = new Set() + for (const snapshot of Object.values(lockfile.packages ?? {})) { + for (const name of Object.keys(snapshot.peerDependencies ?? {})) result.add(name) + for (const name of Object.keys(snapshot.peerDependenciesMeta ?? {})) result.add(name) + for (const name of snapshot.transitivePeerDependencies ?? []) result.add(name) + } + return result +} + +function allResolvedDependencyMaps (lockfile: LockfileObject): ResolvedDependencies[] { + const result: ResolvedDependencies[] = [] + for (const importer of Object.values(lockfile.importers)) { + if (importer.dependencies != null) result.push(importer.dependencies) + if (importer.devDependencies != null) result.push(importer.devDependencies) + if (importer.optionalDependencies != null) result.push(importer.optionalDependencies) + } + for (const snapshot of Object.values(lockfile.packages ?? {})) { + if (snapshot.dependencies != null) result.push(snapshot.dependencies) + if (snapshot.optionalDependencies != null) result.push(snapshot.optionalDependencies) + } + return result +} + +async function resolveNewManifests ( + overrides: FastOverride[], + replacements: Map, + opts: ResolverPolicyOptions & { + lockfileDir: string + parsedOverrides: VersionOverride[] + readPackageHook?: ReadPackageHook + requestPackage: RequestPackageFunction + } +): Promise>['body']['resolution'] +}> | null> { + const changedNames = new Set( + [...replacements] + .filter(([oldDepPath, newDepPath]) => oldDepPath !== newDepPath) + .map(([oldDepPath]) => dp.parse(oldDepPath).name!) + ) + const results = await Promise.all(overrides.map(async ({ name, newVersion }) => { + if (newVersion == null) return null + if (!changedNames.has(name)) return null + const response = await opts.requestPackage({ + alias: name, + bareSpecifier: newVersion, + }, { + downloadPriority: 0, + lockfileDir: opts.lockfileDir, + preferredVersions: Object.create(null), + projectDir: opts.lockfileDir, + publishedBy: opts.publishedBy, + publishedByExclude: opts.publishedByExclude, + skipFetch: true, + trustPolicy: opts.trustPolicy, + trustPolicyExclude: opts.trustPolicyExclude, + trustPolicyIgnoreAfter: opts.trustPolicyIgnoreAfter, + update: false, + }) + if ( + response.body.isLocal || + response.body.manifest == null || + response.body.policyViolation != null || + response.body.resolvedVia !== 'npm-registry' || + response.resolutionNeedsFetch === true || + !('integrity' in response.body.resolution) || + typeof response.body.resolution.integrity !== 'string' || + response.body.resolution.type != null + ) { + return undefined + } + const rawManifest = response.body.manifest + if ( + rawManifest.name !== name || + rawManifest.version !== newVersion || + rawManifest.deprecated != null || + hasInvalidManifestMaps(rawManifest) || + hasPeerDependencies(rawManifest) || + rawManifest.engines?.runtime != null || + rawManifest.bundledDependencies != null || + rawManifest.bundleDependencies != null + ) { + return undefined + } + const manifest = opts.readPackageHook == null + ? rawManifest + : await opts.readPackageHook(clone(rawManifest)) + if (hasPeerDependencies(manifest)) return undefined + return { + name, + manifest, + resolution: response.body.resolution, + } + })) + if (results.some((result) => result === undefined)) return null + return new Map(results + .filter((result) => result != null) + .map(({ name, ...value }) => [name, value])) +} + +function hasInvalidManifestMaps (manifest: PackageManifest): boolean { + return [ + manifest.dependencies, + manifest.optionalDependencies, + manifest.peerDependencies, + manifest.peerDependenciesMeta, + manifest.engines, + ].some((value) => + value != null && + (typeof value !== 'object' || Array.isArray(value)) + ) +} + +function hasPeerDependencies (manifest: PackageManifest): boolean { + return Object.keys(manifest.peerDependencies ?? {}).length > 0 || + Object.keys(manifest.peerDependenciesMeta ?? {}).length > 0 +} + +function rewritePackages ( + originalPackages: Record, + opts: { + lockfileIncludeTarballUrl?: boolean + manifests: Map>['body']['resolution'] + }> + registries: Registries + rewriteContext: RewriteContext + } +): Record | null { + const packages = Object.fromEntries( + Object.entries(originalPackages).map(([depPath, snapshot]) => [ + depPath, + { + ...snapshot, + dependencies: rewriteResolvedDependencies(snapshot.dependencies, opts.rewriteContext, depPath as DepPath), + optionalDependencies: rewriteResolvedDependencies(snapshot.optionalDependencies, opts.rewriteContext, depPath as DepPath), + }, + ]) + ) as Record + + for (const [oldDepPath, newDepPath] of opts.rewriteContext.replacements) { + if (oldDepPath === newDepPath) continue + const oldSnapshot = originalPackages[oldDepPath] + const name = dp.parse(oldDepPath).name! + const resolved = opts.manifests.get(name) + if (resolved == null) return null + const dependencies = validateAndRewriteDependencies({ + lockedDependencies: oldSnapshot.dependencies, + manifestDependencies: effectiveDependencies(resolved.manifest), + packages: originalPackages, + parentDepPath: newDepPath, + rewriteContext: opts.rewriteContext, + }) + const optionalDependencies = validateAndRewriteDependencies({ + lockedDependencies: oldSnapshot.optionalDependencies, + manifestDependencies: resolved.manifest.optionalDependencies, + packages: originalPackages, + parentDepPath: newDepPath, + rewriteContext: opts.rewriteContext, + }) + if (dependencies === null || optionalDependencies === null) return null + + const newSnapshot = createPackageSnapshot(oldSnapshot, { + dependencies, + lockfileIncludeTarballUrl: opts.lockfileIncludeTarballUrl, + manifest: resolved.manifest, + optionalDependencies, + registry: dp.getRegistryByPackageName(opts.registries, name), + resolution: resolved.resolution, + }) + const existingSnapshot = packages[newDepPath] + if (existingSnapshot == null) { + packages[newDepPath] = newSnapshot + } else { + if (!equals(existingSnapshot, newSnapshot)) return null + packages[newDepPath] = newSnapshot + } + } + return packages +} + +function effectiveDependencies (manifest: PackageManifest): Record | undefined { + if (manifest.dependencies == null) return undefined + const optionalNames = new Set(Object.keys(manifest.optionalDependencies ?? {})) + return Object.fromEntries( + Object.entries(manifest.dependencies).filter(([name]) => !optionalNames.has(name)) + ) +} + +function validateAndRewriteDependencies (opts: { + lockedDependencies: ResolvedDependencies | undefined + manifestDependencies: Record | undefined + packages: Record + parentDepPath: DepPath + rewriteContext: RewriteContext +}): ResolvedDependencies | undefined | null { + const { lockedDependencies, manifestDependencies, packages, parentDepPath, rewriteContext } = opts + const manifestEntries = Object.entries(manifestDependencies ?? {}) + for (const name of Object.keys(lockedDependencies ?? {})) { + if (manifestDependencies?.[name] == null && rewriteContext.peerNames.has(name)) return null + } + const result: ResolvedDependencies = {} + for (const [name, range] of manifestEntries) { + if (shouldRemoveDependency(name, parentDepPath, rewriteContext.removals)) continue + if (semver.validRange(range) == null) return null + const lockedReference = lockedDependencies?.[name] + const reference = lockedReference == null + ? findReusableReference({ name, packages, range, rewriteContext }) + : rewriteReference(name, lockedReference, rewriteContext) + if (reference == null) return null + const depPath = dp.refToRelative(reference, name) + const version = depPath == null ? null : dp.parse(depPath).version + if (version == null || !semver.satisfies(version, range)) return null + result[name] = reference + } + return Object.keys(result).length === 0 ? undefined : result +} + +function findReusableReference (opts: { + name: string + packages: Record + range: string + rewriteContext: RewriteContext +}): string | undefined { + const { name, packages, range, rewriteContext } = opts + if ( + rewriteContext.changedNames.has(name) || + rewriteContext.peerNames.has(name) || + rewriteContext.removals.some((removal) => removal.name === name) + ) { + return undefined + } + const candidates = Object.entries(packages) + .filter(([depPath, snapshot]) => { + const parsed = dp.parse(depPath as DepPath) + return parsed.name === name && + parsed.version != null && + parsed.peerDepGraphHash == null && + parsed.patchHash == null && + semver.satisfies(parsed.version, range) && + snapshot.optional !== true && + snapshot.id == null && + snapshot.peerDependencies == null && + snapshot.peerDependenciesMeta == null && + snapshot.transitivePeerDependencies == null && + 'integrity' in snapshot.resolution && + typeof snapshot.resolution.integrity === 'string' && + (!('type' in snapshot.resolution) || snapshot.resolution.type == null) + }) + .map(([depPath]) => depPath as DepPath) + if (candidates.length !== 1) return undefined + return candidates[0].startsWith(`${name}@`) + ? candidates[0].substring(name.length + 1) + : candidates[0] +} + +function createPackageSnapshot ( + oldSnapshot: PackageSnapshot, + opts: { + dependencies?: ResolvedDependencies + lockfileIncludeTarballUrl?: boolean + manifest: PackageManifest + optionalDependencies?: ResolvedDependencies + registry: string + resolution: Awaited>['body']['resolution'] + } +): PackageSnapshot { + const snapshot: PackageSnapshot = { + resolution: toLockfileResolution({ + name: opts.manifest.name, + version: opts.manifest.version, + }, opts.resolution, opts.registry, opts.lockfileIncludeTarballUrl), + } + if (opts.dependencies != null) snapshot.dependencies = opts.dependencies + if (opts.optionalDependencies != null) snapshot.optionalDependencies = opts.optionalDependencies + if (oldSnapshot.optional === true) snapshot.optional = true + if (oldSnapshot.transitivePeerDependencies != null) { + snapshot.transitivePeerDependencies = oldSnapshot.transitivePeerDependencies + } + if (opts.manifest.engines != null) { + const engines = Object.fromEntries( + Object.entries(opts.manifest.engines).filter(([, range]) => range !== '*') + ) + if (Object.keys(engines).length > 0) snapshot.engines = engines as PackageSnapshot['engines'] + } + if (opts.manifest.cpu != null) snapshot.cpu = opts.manifest.cpu + if (opts.manifest.os != null) snapshot.os = opts.manifest.os + if (opts.manifest.libc != null) snapshot.libc = opts.manifest.libc + if (opts.manifest.deprecated) snapshot.deprecated = opts.manifest.deprecated + if (opts.manifest.bin && !(opts.manifest.bin === '' || Object.keys(opts.manifest.bin).length === 0) || opts.manifest.directories?.bin) { + snapshot.hasBin = true + } + return snapshot +} + +function rewriteResolvedDependencies ( + dependencies: ResolvedDependencies | undefined, + rewriteContext: RewriteContext, + parentDepPath?: DepPath +): ResolvedDependencies | undefined { + if (dependencies == null) return undefined + const rewritten = Object.fromEntries( + Object.entries(dependencies) + .filter(([alias]) => !shouldRemoveDependency(alias, parentDepPath, rewriteContext.removals)) + .map(([alias, reference]) => [ + alias, + rewriteReference(alias, reference, rewriteContext), + ]) + ) + return Object.keys(rewritten).length === 0 ? undefined : rewritten +} + +function shouldRemoveDependency ( + alias: string, + parentDepPath: DepPath | undefined, + removals: FastOverride[] +): boolean { + return removals.some((removal) => { + if (removal.name !== alias) return false + if (removal.parent == null) return true + if (parentDepPath == null) return false + const parent = dp.parse(parentDepPath) + return parent.name === removal.parent.name && + parent.version != null && + ( + removal.parent.bareSpecifier == null || + semver.satisfies(parent.version, removal.parent.bareSpecifier) + ) + }) +} + +function rewriteReference ( + alias: string, + reference: string, + { changedNames, replacements }: RewriteContext +): string { + if (!changedNames.has(alias)) return reference + const oldDepPath = dp.refToRelative(reference, alias) + if (oldDepPath == null) return reference + const newDepPath = replacements.get(oldDepPath) + if (newDepPath == null) return reference + return newDepPath.startsWith(`${alias}@`) + ? newDepPath.substring(alias.length + 1) + : newDepPath +} + +function pruneUnreachablePackages ( + importers: LockfileObject['importers'], + packages: Record +): Record { + const reachable = new Set() + const queue: DepPath[] = [] + for (const importer of Object.values(importers)) { + enqueueDependencies(importer.dependencies) + enqueueDependencies(importer.devDependencies) + enqueueDependencies(importer.optionalDependencies) + } + for (let index = 0; index < queue.length; index++) { + const depPath = queue[index] + const snapshot = packages[depPath] + if (snapshot == null) continue + enqueueDependencies(snapshot.dependencies) + enqueueDependencies(snapshot.optionalDependencies) + } + return Object.fromEntries( + Object.entries(packages).filter(([depPath]) => reachable.has(depPath as DepPath)) + ) as Record + + function enqueueDependencies (dependencies: ResolvedDependencies | undefined): void { + for (const [alias, reference] of Object.entries(dependencies ?? {})) { + const depPath = dp.refToRelative(reference, alias) + if (depPath == null || reachable.has(depPath)) continue + reachable.add(depPath) + queue.push(depPath) + } + } +} diff --git a/pnpm11/installing/deps-installer/test/install/overrides.ts b/pnpm11/installing/deps-installer/test/install/overrides.ts index ae3facfbf7..b12e562ab5 100644 --- a/pnpm11/installing/deps-installer/test/install/overrides.ts +++ b/pnpm11/installing/deps-installer/test/install/overrides.ts @@ -1,18 +1,393 @@ import fs from 'node:fs' import path from 'node:path' -import { expect, test } from '@jest/globals' +import { expect, jest, test } from '@jest/globals' import { WANTED_LOCKFILE } from '@pnpm/constants' import { PnpmError } from '@pnpm/error' import { addDependenciesToPackage, type MutatedProject, mutateModules, mutateModulesInSingleProject, type ProjectOptions } from '@pnpm/installing.deps-installer' import type { LockfileFile } from '@pnpm/lockfile.types' import { prepare, prepareEmpty, preparePackages } from '@pnpm/prepare' +import type { ResolutionVerifier } from '@pnpm/resolving.resolver-base' +import type { RequestPackageOptions, StoreController } from '@pnpm/store.controller-types' import { addDistTag } from '@pnpm/testing.registry-mock' import type { ProjectManifest, ProjectRootDir } from '@pnpm/types' import { readYamlFileSync } from 'read-yaml-file' import { testDefaults } from '../utils/index.js' +function trackRequestedPackages ( + storeController: StoreController, + onRequest?: (requestOptions: RequestPackageOptions) => void +): string[] { + const requestedPackages: string[] = [] + const requestPackage = storeController.requestPackage + storeController.requestPackage = async (wantedDependency, requestOptions) => { + requestedPackages.push(wantedDependency.alias!) + onRequest?.(requestOptions) + return requestPackage(wantedDependency, requestOptions) + } + return requestedPackages +} + +test('adding an exact override reuses the lockfile when the new package has the same dependencies', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/foobarqar': '1.0.0', + }, + } + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/bar': '100.1.0', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/bar']) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.0'].dependencies?.['@pnpm.e2e/bar']).toBe('100.1.0') +}) + +test('an exact override update preserves resolver trust policies', async () => { + prepareEmpty() + const reporter = jest.fn() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/foobarqar': '1.0.0', + }, + } + const options = testDefaults({ + handleResolutionPolicyViolations: async () => {}, + hooks: { + afterAllResolved: [], + preResolution: [], + readPackage: [], + }, + reporter, + trustPolicy: 'no-downgrade', + trustPolicyExclude: ['@pnpm.e2e/bar'], + }) + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + reporter.mockClear() + const verify = jest.fn(async () => ({ ok: true })) + options.resolutionVerifiers = [{ + canTrustPastCheck: () => false, + policy: { test: true }, + verify, + }] + const requestOptions: RequestPackageOptions[] = [] + const requestedPackages = trackRequestedPackages( + options.storeController, + (options) => requestOptions.push(options) + ) + options.overrides = { + '@pnpm.e2e/bar': '100.1.0', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/bar']) + expect(requestOptions[0].trustPolicy).toBe('no-downgrade') + expect(requestOptions[0].trustPolicyExclude?.('@pnpm.e2e/bar')).toBe(true) + expect(reporter).not.toHaveBeenCalledWith(expect.objectContaining({ + name: 'pnpm:stage', + stage: 'resolution_started', + })) + expect(verify).toHaveBeenCalled() + expect(verify.mock.calls).not.toContainEqual([ + expect.anything(), + expect.objectContaining({ name: '@pnpm.e2e/bar', version: '100.1.0' }), + ]) +}) + +test('an exact override update reuses the lockfile when the new package has the same dependencies', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/parent-of-pkg-with-1-dep': '1.0.0', + }, + } + const options = testDefaults({ + overrides: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + }, + }) + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const previousLockfile = project.readLockfile() + const previousChildResolution = previousLockfile.snapshots['@pnpm.e2e/pkg-with-1-dep@100.0.0'] + .dependencies?.['@pnpm.e2e/dep-of-pkg-with-1-dep'] + expect(previousChildResolution).toBeDefined() + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/pkg-with-1-dep': '100.1.0', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/pkg-with-1-dep']) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/pkg-with-1-dep@100.1.0'] + .dependencies?.['@pnpm.e2e/dep-of-pkg-with-1-dep']).toBe(previousChildResolution) + expect(lockfile.packages).toHaveProperty(['@pnpm.e2e/pkg-with-1-dep@100.1.0']) + expect(lockfile.packages).not.toHaveProperty(['@pnpm.e2e/pkg-with-1-dep@100.0.0']) + const currentLockfile = project.readCurrentLockfile() + expect(currentLockfile.snapshots['@pnpm.e2e/pkg-with-1-dep@100.1.0'] + .dependencies?.['@pnpm.e2e/dep-of-pkg-with-1-dep']).toBe(previousChildResolution) +}) + +test('an exact override update falls back to resolution when the package dependencies changed', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/foobarqar': '^1.0.0', + }, + } + const options = testDefaults({ + overrides: { + '@pnpm.e2e/foobarqar': '1.0.0', + }, + }) + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/foobarqar': '1.0.1', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toContain('@pnpm.e2e/qar') + const lockfile = project.readLockfile() + expect(lockfile.snapshots).toHaveProperty(['@pnpm.e2e/foobarqar@1.0.1']) + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.1'].dependencies).toHaveProperty(['@pnpm.e2e/qar']) + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.1'].dependencies).not.toHaveProperty(['is-positive']) +}) + +test('a dependency removal override prunes the locked subtree without resolution', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/pkg-with-good-optional': '1.0.0', + }, + } + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + 'is-positive': '-', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual([]) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/pkg-with-good-optional@1.0.0']) + .not.toHaveProperty(['optionalDependencies', 'is-positive']) + expect(lockfile.snapshots).not.toHaveProperty(['is-positive@1.0.0']) + expect(lockfile.packages).not.toHaveProperty(['is-positive@1.0.0']) + const currentLockfile = project.readCurrentLockfile() + expect(currentLockfile.snapshots['@pnpm.e2e/pkg-with-good-optional@1.0.0']) + .not.toHaveProperty(['optionalDependencies', 'is-positive']) + expect( + fs.existsSync('node_modules/.pnpm/@pnpm.e2e+pkg-with-good-optional@1.0.0/node_modules/is-positive') + ).toBe(false) +}) + +test('a parent-scoped dependency removal override only prunes matching edges', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/pkg-with-good-optional': '1.0.0', + 'is-positive': '1.0.0', + }, + } + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/pkg-with-good-optional@1>is-positive': '-', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual([]) + const lockfile = project.readLockfile() + expect(lockfile.importers['.'].dependencies).toHaveProperty(['is-positive']) + expect(lockfile.snapshots['@pnpm.e2e/pkg-with-good-optional@1.0.0']) + .not.toHaveProperty(['optionalDependencies', 'is-positive']) + expect(lockfile.snapshots).toHaveProperty(['is-positive@1.0.0']) +}) + +test('exact replacements and dependency removals reuse the lockfile together', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/foobarqar': '1.0.0', + }, + } + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/bar': '100.1.0', + 'is-positive': '-', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/bar']) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.0'].dependencies?.['@pnpm.e2e/bar']).toBe('100.1.0') + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.0'].dependencies).not.toHaveProperty(['is-positive']) + expect(lockfile.snapshots).not.toHaveProperty(['is-positive@1.0.0']) +}) + +test('a dependency removal also applies to snapshots rebuilt for replacements', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/parent-of-foobarqar': '1.0.1', + }, + } + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/foobarqar': '1.0.1', + '@pnpm.e2e/qar': '-', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/foobarqar']) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/parent-of-foobarqar@1.0.1'].dependencies) + .not.toHaveProperty(['@pnpm.e2e/qar']) + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.1'].dependencies) + .not.toHaveProperty(['@pnpm.e2e/qar']) + expect(lockfile.snapshots).not.toHaveProperty(['@pnpm.e2e/qar@100.0.0']) +}) + +test('an exact override update reuses uniquely compatible locked dependencies and drops obsolete edges', async () => { + const project = prepareEmpty() + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/parent-of-foobarqar': '1.0.0', + '@pnpm.e2e/qar': '100.0.0', + }, + } + const options = testDefaults({ + overrides: { + '@pnpm.e2e/foobarqar': '1.0.0', + }, + }) + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const requestedPackages = trackRequestedPackages(options.storeController) + options.overrides = { + '@pnpm.e2e/foobarqar': '1.0.1', + } + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(requestedPackages).toStrictEqual(['@pnpm.e2e/foobarqar']) + const lockfile = project.readLockfile() + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.1'].dependencies?.['@pnpm.e2e/qar']).toBe('100.0.0') + expect(lockfile.snapshots['@pnpm.e2e/foobarqar@1.0.1'].dependencies).not.toHaveProperty(['is-positive']) + expect(lockfile.snapshots).not.toHaveProperty(['is-positive@1.0.0']) + expect(lockfile.packages).not.toHaveProperty(['is-positive@1.0.0']) +}) + test('versions are replaced with versions specified through overrides option', async () => { const project = prepareEmpty() diff --git a/pnpm11/installing/deps-installer/test/install/relinkingScope.test.ts b/pnpm11/installing/deps-installer/test/install/relinkingScope.test.ts index 92f71670c7..349e250522 100644 --- a/pnpm11/installing/deps-installer/test/install/relinkingScope.test.ts +++ b/pnpm11/installing/deps-installer/test/install/relinkingScope.test.ts @@ -82,6 +82,36 @@ test('relinks only changed child edges for existing packages after dependency up expect(pkgCalls.every(({ children }) => !children.includes('is-positive'))).toBe(true) }) +test('restores an unchanged child edge when its on-disk link is missing', async () => { + const manifest: ProjectManifest = { + dependencies: { + '@pnpm.e2e/pkg-with-good-optional': '1.0.0', + }, + } + prepare(manifest) + const options = testDefaults() + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + const childLink = path.resolve('node_modules/.pnpm/@pnpm.e2e+pkg-with-good-optional@1.0.0/node_modules/is-positive') + fs.rmSync(childLink, { recursive: true }) + expect(fs.existsSync(childLink)).toBe(false) + symlinkAllModulesCalls.length = 0 + + await mutateModulesInSingleProject({ + manifest, + mutation: 'install', + rootDir: process.cwd() as ProjectRootDir, + }, options) + + expect(fs.realpathSync(childLink)).toContain('is-positive@1.0.0') + expect(symlinkAllModulesCalls.flat().some(({ children }) => children.includes('is-positive'))).toBe(true) +}) + test('removes obsolete child links for existing packages after dependency updates', async () => { const manifest: ProjectManifest = { dependencies: { diff --git a/pnpm11/installing/deps-restorer/src/index.ts b/pnpm11/installing/deps-restorer/src/index.ts index f6c790f725..85178a105e 100644 --- a/pnpm11/installing/deps-restorer/src/index.ts +++ b/pnpm11/installing/deps-restorer/src/index.ts @@ -30,10 +30,10 @@ import { makeNodeRequireOption, runLifecycleHooksConcurrently, } from '@pnpm/exec.lifecycle' -import { symlinkDependency } from '@pnpm/fs.symlink-dependency' +import { safeJoinModulesDir, symlinkDependency } from '@pnpm/fs.symlink-dependency' import { linkDirectDeps, type LinkedDirectDep } from '@pnpm/installing.linking.direct-dep-linker' import { hoist, type HoistedWorkspaceProject } from '@pnpm/installing.linking.hoist' -import { prune } from '@pnpm/installing.linking.modules-cleaner' +import { prune, removeObsoleteDependency } from '@pnpm/installing.linking.modules-cleaner' import type { HoistingLimits } from '@pnpm/installing.linking.real-hoist' import { type IncludedDependencies, @@ -175,6 +175,7 @@ export interface HeadlessOptions { wantedLockfile?: LockfileObject ownLifecycleHooksStdio?: 'inherit' | 'pipe' pendingBuilds: string[] + relinkChangedDependenciesOnly?: boolean resolveSymlinksInInjectedDirs?: boolean skipped: Set skipRuntimes?: boolean @@ -457,7 +458,9 @@ export async function headlessInstall (opts: HeadlessOptions): Promise + async function linkAllModules ( - depNodes: Array>, + depNodes: ModulesLinkNode[], opts: { + currentLockfile?: LockfileObject | null optional: boolean + wantedLockfile: LockfileObject } ): Promise { + const changes = await Promise.all(depNodes.map((depNode) => getChangedChildren(depNode, opts))) + await Promise.all(changes.flatMap(({ depNode, removedAliases }) => + removedAliases.map((alias) => limitModulesDirReads(() => removeObsoleteDependency(depNode.modules, alias))) + )) await symlinkAllModules({ - deps: depNodes.map((depNode) => ({ - children: opts.optional - ? depNode.children - : pickBy((_, childAlias) => !depNode.optionalDependencies.has(childAlias), depNode.children), - modules: depNode.modules, - name: depNode.name, - })), + deps: changes.map(({ children, depNode }) => { + return { + children: opts.optional + ? children + : pickBy((_, childAlias) => !depNode.optionalDependencies.has(childAlias), children), + modules: depNode.modules, + name: depNode.name, + } + }), }) } + +async function getChangedChildren ( + depNode: ModulesLinkNode, + opts: { + currentLockfile?: LockfileObject | null + wantedLockfile: LockfileObject + } +): Promise<{ + children: Record + depNode: ModulesLinkNode + removedAliases: string[] +}> { + const currentSnapshot = opts.currentLockfile?.packages?.[depNode.depPath] + const wantedSnapshot = opts.wantedLockfile.packages?.[depNode.depPath] + if (currentSnapshot == null || wantedSnapshot == null) { + return { children: depNode.children, depNode, removedAliases: [] } + } + const currentDependencies = Object.assign(Object.create(null), currentSnapshot.dependencies, currentSnapshot.optionalDependencies) as Record + const wantedDependencies = Object.assign(Object.create(null), wantedSnapshot.dependencies, wantedSnapshot.optionalDependencies) as Record + const changedChildren = Object.fromEntries( + (await Promise.all(Object.entries(depNode.children).map(async ([alias, childDir]) => { + if ( + currentDependencies[alias] !== wantedDependencies[alias] || + Object.hasOwn(currentSnapshot.optionalDependencies ?? {}, alias) !== Object.hasOwn(wantedSnapshot.optionalDependencies ?? {}, alias) || + !await limitModulesDirReads(() => dependencyLinkMatches(depNode.modules, alias, childDir)) + ) { + return [alias, childDir] as const + } + return null + }))).filter((entry): entry is readonly [string, string] => entry != null) + ) + return { + children: changedChildren, + depNode, + removedAliases: Object.keys(currentDependencies).filter((alias) => !Object.hasOwn(wantedDependencies, alias)), + } +} + +async function dependencyLinkMatches (modulesDir: string, alias: string, childDir: string): Promise { + const [linkTarget, expectedTarget] = await Promise.all([ + realpathOrNull(safeJoinModulesDir(modulesDir, alias)), + realpathOrNull(childDir), + ]) + return linkTarget != null && expectedTarget != null && linkTarget === expectedTarget +} + +async function realpathOrNull (filePath: string): Promise { + try { + return await fs.realpath(filePath) + } catch (err: unknown) { + if ( + typeof err === 'object' && + err != null && + 'code' in err && + (err.code === 'ENOENT' || err.code === 'ENOTDIR' || err.code === 'ELOOP') + ) { + return null + } + throw err + } +} diff --git a/pnpm11/installing/linking/modules-cleaner/package.json b/pnpm11/installing/linking/modules-cleaner/package.json index fcba43dcf1..4699466ff6 100644 --- a/pnpm11/installing/linking/modules-cleaner/package.json +++ b/pnpm11/installing/linking/modules-cleaner/package.json @@ -41,7 +41,8 @@ "@pnpm/store.controller-types": "workspace:*", "@pnpm/types": "workspace:*", "@zkochan/rimraf": "catalog:", - "ramda": "catalog:" + "ramda": "catalog:", + "validate-npm-package-name": "catalog:" }, "peerDependencies": { "@pnpm/logger": "catalog:" @@ -49,7 +50,8 @@ "devDependencies": { "@pnpm/installing.linking.modules-cleaner": "workspace:*", "@pnpm/logger": "workspace:*", - "@types/ramda": "catalog:" + "@types/ramda": "catalog:", + "@types/validate-npm-package-name": "catalog:" }, "engines": { "node": ">=22.13" diff --git a/pnpm11/installing/linking/modules-cleaner/src/index.ts b/pnpm11/installing/linking/modules-cleaner/src/index.ts index ff986df569..997aaade9a 100644 --- a/pnpm11/installing/linking/modules-cleaner/src/index.ts +++ b/pnpm11/installing/linking/modules-cleaner/src/index.ts @@ -1 +1,2 @@ export { prune } from './prune.js' +export { removeObsoleteDependency } from './removeObsoleteDependency.js' diff --git a/pnpm11/installing/linking/modules-cleaner/src/removeObsoleteDependency.ts b/pnpm11/installing/linking/modules-cleaner/src/removeObsoleteDependency.ts new file mode 100644 index 0000000000..149accc129 --- /dev/null +++ b/pnpm11/installing/linking/modules-cleaner/src/removeObsoleteDependency.ts @@ -0,0 +1,24 @@ +import { promises as fs } from 'node:fs' +import path from 'node:path' +import util from 'node:util' + +import { rimraf } from '@zkochan/rimraf' +import validateNpmPackageName from 'validate-npm-package-name' + +export async function removeObsoleteDependency (modulesDir: string, alias: string): Promise { + if (!validateNpmPackageName(alias).validForOldPackages) return + await rimraf(path.join(modulesDir, alias)) + if (alias[0] === '@') { + try { + await fs.rmdir(path.join(modulesDir, alias.split('/')[0])) + } catch (err: unknown) { + if ( + !util.types.isNativeError(err) || + !('code' in err) || + err.code !== 'ENOENT' && err.code !== 'ENOTEMPTY' + ) { + throw err + } + } + } +} diff --git a/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.0/package.json b/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.0/package.json new file mode 100644 index 0000000000..07e75244fd --- /dev/null +++ b/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.0/package.json @@ -0,0 +1,7 @@ +{ + "name": "@pnpm.e2e/parent-of-foobarqar", + "version": "1.0.0", + "dependencies": { + "@pnpm.e2e/foobarqar": "^1.0.0" + } +} diff --git a/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.1/package.json b/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.1/package.json new file mode 100644 index 0000000000..bff8cabd18 --- /dev/null +++ b/pnpr/.fixtures/packages/@pnpm.e2e/parent-of-foobarqar/1.0.1/package.json @@ -0,0 +1,8 @@ +{ + "name": "@pnpm.e2e/parent-of-foobarqar", + "version": "1.0.1", + "dependencies": { + "@pnpm.e2e/foobarqar": "1.0.0", + "@pnpm.e2e/qar": "100.0.0" + } +}