diff --git a/.changeset/store-prune-keeps-copied-packages.md b/.changeset/store-prune-keeps-copied-packages.md new file mode 100644 index 0000000000..2e502349b0 --- /dev/null +++ b/.changeset/store-prune-keeps-copied-packages.md @@ -0,0 +1,8 @@ +--- +"@pnpm/lockfile.fs": patch +"@pnpm/store.controller": patch +"pnpm": patch +"pacquet": patch +--- + +`pnpm store prune` now keeps the packages that a project still uses when pnpm copied or cloned them into `node_modules`. These include patched packages, built packages, and every package installed with `packageImportMethod: copy`. Before, the prune removed them, and the next install downloaded them again [#7192](https://github.com/pnpm/pnpm/issues/7192). diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a1adaf890..ed4aa0ed2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10173,6 +10173,9 @@ importers: pnpm11/store/controller: dependencies: + '@pnpm/constants': + specifier: workspace:* + version: link:../../core/constants '@pnpm/crypto.hash': specifier: workspace:* version: link:../../crypto/hash @@ -10185,9 +10188,21 @@ importers: '@pnpm/hooks.types': specifier: workspace:* version: link:../../hooks/types + '@pnpm/installing.modules-yaml': + specifier: workspace:* + version: link:../../installing/modules-yaml '@pnpm/installing.package-requester': specifier: workspace:* version: link:../../installing/package-requester + '@pnpm/lockfile.fs': + specifier: workspace:* + version: link:../../lockfile/fs + '@pnpm/lockfile.types': + specifier: workspace:* + version: link:../../lockfile/types + '@pnpm/lockfile.utils': + specifier: workspace:* + version: link:../../lockfile/utils '@pnpm/resolving.resolver-base': specifier: workspace:* version: link:../../resolving/resolver-base @@ -10209,9 +10224,15 @@ importers: '@pnpm/worker': specifier: workspace:^ version: link:../../worker + '@pnpm/workspace.workspace-manifest-reader': + specifier: workspace:* + version: link:../../workspace/workspace-manifest-reader '@zkochan/rimraf': specifier: 'catalog:' version: 4.0.0 + empathic: + specifier: 'catalog:' + version: 2.1.0 is-subdir: specifier: 'catalog:' version: 2.0.1 diff --git a/pnpm/crates/cli/src/cli_args/store.rs b/pnpm/crates/cli/src/cli_args/store.rs index d58545728b..afe013f291 100644 --- a/pnpm/crates/cli/src/cli_args/store.rs +++ b/pnpm/crates/cli/src/cli_args/store.rs @@ -7,6 +7,7 @@ //! any project. mod add; +mod in_use_files; mod status; use crate::{ @@ -80,7 +81,11 @@ fn prune(config: &Config) -> miette::Result<()> { .into_iter() .flatten() .collect(); - config.store_dir.prune(&config.cache_dir, &protected_dirs).wrap_err("pruning store")?; + config.store_dir + .prune_keeping(&config.cache_dir, &protected_dirs, || { + in_use_files::in_use_files(&config.store_dir) + }) + .wrap_err("pruning store")?; remove_orphaned_private_engine_installs(config)?; Ok(()) } diff --git a/pnpm/crates/cli/src/cli_args/store/in_use_files.rs b/pnpm/crates/cli/src/cli_args/store/in_use_files.rs new file mode 100644 index 0000000000..5eebfe664e --- /dev/null +++ b/pnpm/crates/cli/src/cli_args/store/in_use_files.rs @@ -0,0 +1,178 @@ +//! The store files of the packages that registered projects installed. +//! +//! A package imported by copy or clone, such as a patched or built one, has +//! no hard link to the store, so `store prune` needs this list to keep it. + +use pnpm_config::workspace_root_or; +use pnpm_deps_restorer::{binary_store_index_key, store_index_key_for_resolution}; +use pnpm_fs::lexical_normalize; +use pnpm_lockfile::{Lockfile, LockfileResolution, PackageKey}; +use pnpm_modules_yaml::{Host, read_modules_layout}; +use pnpm_store_dir::{ + CafsFileInfo, PackageFilesIndex, PruneError, StoreDir, StoreIndex, StoreIndexError, + decode_package_files_index, get_registered_projects, +}; +use pnpm_workspace::read_workspace_manifest_document; +use serde::Deserialize; +use std::{ + collections::HashSet, + io, iter, + path::{Path, PathBuf}, +}; + +/// Every file of the store index rows of the packages in each registered +/// project's current lockfile, including the files of every side-effects +/// entry. A project without a current lockfile, a project whose lockfile +/// version this pnpm does not support, and a package without a row add +/// nothing. A registry error keeps its own code, and any other +/// failure is [`PruneError::InUseFiles`]. +pub(super) fn in_use_files(store_dir: &StoreDir) -> Result, PruneError> { + if !StoreIndex::exists(store_dir.root()) { + return Ok(HashSet::new()); + } + let mut keys = HashSet::new(); + for project in get_registered_projects(store_dir).map_err(PruneError::ListProjects)? { + // The registry joins a relative link target without normalizing it. + let project = lexical_normalize(&project); + for lockfile in load_installed_lockfiles(&project).map_err(PruneError::InUseFiles)? { + keys.extend(package_index_keys(&lockfile).map_err(PruneError::InUseFiles)?); + } + } + if keys.is_empty() { + return Ok(HashSet::new()); + } + let keys: Vec = keys.into_iter().collect(); + let rows = read_rows(store_dir, &keys) + .map_err(|error| PruneError::InUseFiles(io::Error::other(error)))?; + Ok(rows + .iter() + .flat_map(row_files) + .filter_map(|file| store_dir.cas_file_path_by_mode(&file.digest, file.mode)) + .collect()) +} + +/// Fails on an undecodable row, which [`StoreIndex::get_many`] skips, +/// because skipping it would let the prune remove the row's files. +fn read_rows( + store_dir: &StoreDir, + keys: &[String], +) -> Result, StoreIndexError> { + StoreIndex::open_readonly_in(store_dir)? + .get_many_raw(keys)? + .iter() + .map(|(_, bytes)| decode_package_files_index(bytes)) + .collect() +} + +/// The lockfiles the last install wrote, which record what is installed in +/// the project. +/// +/// The install writes the lockfile to `/.pnpm`, or to the +/// `virtualStoreDir` that `/.modules.yaml` records when the +/// project sets one. Under a global virtual store, that setting names the +/// shared store and the lockfile stays in `/.pnpm`. The +/// manifest does not say which case applies, so the prune reads both. +fn load_installed_lockfiles(project: &Path) -> io::Result> { + let modules_dir = project.join(configured_modules_dir(project)?); + let default_dir = modules_dir.join(".pnpm"); + let virtual_store_dir = read_modules_layout::(&modules_dir) + .map_err(io::Error::other)? + .map(|layout| PathBuf::from(layout.virtual_store_dir)) + .filter(|dir| *dir != default_dir); + let mut lockfiles = Vec::new(); + for install_state_dir in iter::once(default_dir).chain(virtual_store_dir) { + lockfiles.extend(load_installed_lockfile(project, &install_state_dir)?); + } + Ok(lockfiles) +} + +/// The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the +/// project sets, or `node_modules`. A `modulesDir` set only on the command +/// line, in the environment, in the global config, or in `packageConfigs` +/// is not seen. +fn configured_modules_dir(project: &Path) -> io::Result { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct ModulesDirSetting { + modules_dir: Option, + } + let Some(document) = + read_workspace_manifest_document(&workspace_root_or(project)).map_err(io::Error::other)? + else { + return Ok("node_modules".to_string()); + }; + let setting: Option = serde_saphyr::from_str(&document.text) + .map_err(|error| io::Error::other(format!("{}: {error}", document.path.display())))?; + Ok(setting + .and_then(|setting| setting.modules_dir) + .unwrap_or_else(|| "node_modules".to_string())) +} + +/// A lockfile of a version this pnpm cannot read adds nothing, like a +/// missing one, so one project installed by another pnpm does not stop the +/// prune. The hard-link check then decides for that project's files. +fn load_installed_lockfile( + project: &Path, + install_state_dir: &Path, +) -> io::Result> { + let error = match Lockfile::load_current_from_install_state_dir(install_state_dir) { + Ok(lockfile) => return Ok(lockfile), + Err(error) => error, + }; + let lockfile_path = install_state_dir.join(Lockfile::CURRENT_FILE_NAME); + if let Ok(Some(version)) = Lockfile::unsupported_version_at(&lockfile_path) { + pnpm_diagnostics::eprintln!( + "Skipped {} because it was installed with lockfileVersion {version}, \ + which this version of pnpm does not support", + project.display(), + ); + return Ok(None); + } + Err(io::Error::other(format!("{}: {error}", install_state_dir.display()))) +} + +fn package_index_keys(lockfile: &Lockfile) -> io::Result> { + let mut keys = Vec::new(); + for (key, metadata) in lockfile.packages.iter().flatten() { + add_resolution_index_keys(&mut keys, key, &metadata.resolution)?; + } + Ok(keys) +} + +/// The lockfile does not say which key the install used for: +/// - a git-hosted package, which has a key for its built and its unbuilt +/// contents; +/// - a `Variations` resolution, whose variant depends on the installing +/// host and its `supportedArchitectures`; +/// - a runtime, whose row an older install keyed without its manifest. +fn add_resolution_index_keys( + keys: &mut Vec, + key: &PackageKey, + resolution: &LockfileResolution, +) -> io::Result<()> { + if let LockfileResolution::Variations(variations) = resolution { + for variant in &variations.variants { + add_resolution_index_keys(keys, key, &variant.resolution)?; + } + return Ok(()); + } + let pkg_id = key.pkg_id(); + keys.extend( + [true, false] + .into_iter() + .filter_map(|built| store_index_key_for_resolution(resolution, &pkg_id, built)), + ); + if let LockfileResolution::Binary(binary) = resolution { + keys.push(binary_store_index_key(key, binary).map_err(io::Error::other)?); + } + Ok(()) +} + +fn row_files(row: &PackageFilesIndex) -> impl Iterator { + let side_effects_files = row.side_effects + .iter() + .flat_map(|side_effects| side_effects.values()) + .filter_map(|diff| diff.added.as_ref()) + .flat_map(|added| added.values()); + row.files.values().chain(side_effects_files) +} diff --git a/pnpm/crates/cli/tests/suite/install_runtimes.rs b/pnpm/crates/cli/tests/suite/install_runtimes.rs index b38f449040..8304885669 100644 --- a/pnpm/crates/cli/tests/suite/install_runtimes.rs +++ b/pnpm/crates/cli/tests/suite/install_runtimes.rs @@ -13,7 +13,7 @@ use std::{ }; use tempfile::TempDir; -struct RuntimeFixture { +pub(crate) struct RuntimeFixture { name: &'static str, version: &'static str, archive_mock: mockito::Mock, @@ -963,7 +963,7 @@ fn assert_runtime_bad_integrity(name: &'static str, version: &'static str) { assert!(stderr.contains("Integrity check failed"), "stderr was:\n{stderr}"); } -fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf { +pub(crate) fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf { let workspace = root.path().join("workspace"); fs::create_dir(&workspace).unwrap(); fs::write( @@ -976,7 +976,7 @@ fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf { workspace } -fn runtime_fixture( +pub(crate) fn runtime_fixture( server: &mut mockito::Server, name: &'static str, version: &'static str, @@ -1027,7 +1027,7 @@ fn runtime_fixture( } } -fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) { +pub(crate) fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) { let dependencies = fixtures .iter() .map(|fixture| { @@ -1038,7 +1038,7 @@ fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) { .unwrap(); } -fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) { +pub(crate) fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) { write_runtime_lockfile_for_group(workspace, fixtures, "dependencies"); } diff --git a/pnpm/crates/cli/tests/suite/main.rs b/pnpm/crates/cli/tests/suite/main.rs index 57cbb287ae..9247930f70 100644 --- a/pnpm/crates/cli/tests/suite/main.rs +++ b/pnpm/crates/cli/tests/suite/main.rs @@ -192,6 +192,7 @@ mod store; mod store_pkg_content_check; mod store_placement; mod store_project_registry; +mod store_prune_in_use; mod sync_injected_deps_after_scripts; mod tar_registry; mod tarball_url_dependency; diff --git a/pnpm/crates/cli/tests/suite/store_prune_in_use.rs b/pnpm/crates/cli/tests/suite/store_prune_in_use.rs new file mode 100644 index 0000000000..714fb7b810 --- /dev/null +++ b/pnpm/crates/cli/tests/suite/store_prune_in_use.rs @@ -0,0 +1,436 @@ +//! `pnpm store prune` must keep the store files of every package a +//! registered project installed, including the packages that pnpm copied +//! into `node_modules`, which hold no hard link to the store. + +use crate::{ + _utils::enable_gvs_in_workspace_yaml, + install_runtimes::{ + prepare_workspace, runtime_fixture, write_runtime_lockfile, write_runtime_manifest, + }, + patch::MARKER_PATCH, +}; +use assert_cmd::prelude::*; +use command_extra::CommandExtra; +use pnpm_graph_hasher::{host_arch, host_platform}; +use pnpm_store_dir::{PackageFilesIndex, StoreDir, StoreIndex}; +use pnpm_testing_utils::{ + bin::{AddMockedRegistry, CommandTempCwd}, + command_env::CommandTestExt, +}; +use pretty_assertions::assert_eq; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +fn pacquet_at(workspace: &Path) -> Command { + Command::cargo_bin("pnpm") + .expect("find the pnpm binary") + .with_current_dir(workspace) + .without_ambient_pnpm_config() +} + +fn write_manifest(workspace: &Path) { + fs::write( + workspace.join("package.json"), + serde_json::json!({ + "dependencies": { + "@pnpm.e2e/pkg-with-1-dep": "100.0.0", + "is-positive": "1.0.0", + }, + }) + .to_string(), + ) + .expect("write package.json"); +} + +fn append_workspace_yaml(workspace: &Path, text: &str) { + let path = workspace.join("pnpm-workspace.yaml"); + let mut yaml = fs::read_to_string(&path).expect("read pnpm-workspace.yaml"); + if !yaml.ends_with('\n') { + yaml.push('\n'); + } + yaml.push_str(text); + fs::write(&path, yaml).expect("write pnpm-workspace.yaml"); +} + +fn sorted_index_keys(store_dir: &StoreDir) -> Vec { + let mut keys = StoreIndex::open_readonly_in(store_dir) + .expect("open the store index") + .keys() + .expect("read the store index keys"); + keys.sort(); + keys +} + +fn is_positive_row(store_dir: &StoreDir) -> PackageFilesIndex { + let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index"); + let key = index + .keys() + .expect("read the store index keys") + .into_iter() + .find(|key| key.ends_with("\tis-positive@1.0.0")) + .expect("a store index row for is-positive@1.0.0"); + index + .get(&key) + .expect("read the store index row") + .expect("the row is present") +} + +/// Every store file the row names: its base files and the files each +/// side-effects entry adds. +fn row_blob_paths(store_dir: &StoreDir, row: &PackageFilesIndex) -> Vec { + let side_effects_files = row.side_effects + .iter() + .flat_map(|side_effects| side_effects.values()) + .filter_map(|diff| diff.added.as_ref()) + .flat_map(|added| added.values()); + row.files + .values() + .chain(side_effects_files) + .map(|info| { + store_dir + .cas_file_path_by_mode(&info.digest, info.mode) + .expect("an index row holds hex digests") + }) + .collect() +} + +/// Every store file the rows under `keys` name. +fn blob_paths_of_rows(store_dir: &StoreDir, keys: &[String]) -> Vec { + let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index"); + keys.iter() + .flat_map(|key| { + let row = index + .get(key) + .expect("read the store index row") + .expect("the row is present"); + row_blob_paths(store_dir, &row) + }) + .collect() +} + +fn prune(workspace: &Path) { + let output = pacquet_at(workspace) + .with_args(["store", "prune"]) + .output() + .expect("run store prune"); + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("STDERR:\n{stderr}\n"); + assert!(output.status.success(), "store prune failed: {stderr}"); +} + +/// A patched package gets its own copy of the files even under `hardlink`, +/// so no store file of it has a hard link outside the store. `hardlink` +/// keeps every other package linked on each platform, which `auto` does +/// not do on a copy-on-write filesystem. +#[test] +fn store_prune_keeps_the_files_of_a_patched_dependency() { + let CommandTempCwd { + root: _root, workspace, npmrc_info, .. + } = CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { + mock_instance: _mock_instance, + store_dir, + .. + } = npmrc_info; + write_manifest(&workspace); + fs::create_dir_all(workspace.join("patches")).expect("create the patches dir"); + fs::write(workspace.join("patches/is-positive@1.0.0.patch"), MARKER_PATCH) + .expect("write the patch file"); + append_workspace_yaml( + &workspace, + concat!( + "packageImportMethod: hardlink\n", + "patchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n", + ), + ); + pacquet_at(&workspace) + .with_args(["install", "--reporter=silent"]) + .assert() + .success(); + + let store_dir = StoreDir::new(store_dir); + let row = is_positive_row(&store_dir); + let patched_entries = row.side_effects + .as_ref() + .map_or(0, |side_effects| { + side_effects + .keys() + .filter(|key| key.contains(";patch=")) + .count() + }); + assert_eq!(patched_entries, 1, "the patch must be cached as a side-effects entry"); + let blob_paths = row_blob_paths(&store_dir, &row); + let keys_before = sorted_index_keys(&store_dir); + + prune(&workspace); + + assert_eq!(sorted_index_keys(&store_dir), keys_before); + let missing: Vec<_> = blob_paths + .iter() + .filter(|path| !path.exists()) + .collect(); + assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses"); +} + +/// With `packageImportMethod: copy` no store file has a hard link outside +/// the store, so the hard-link count alone marks every package unused. +#[test] +fn store_prune_keeps_every_package_installed_by_copy() { + assert_prune_keeps_every_package_installed_by_copy(|_| {}, "node_modules/.pnpm/lock.yaml"); +} + +#[test] +fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir() { + assert_prune_keeps_every_package_installed_by_copy( + |workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\n"), + "vs/lock.yaml", + ); +} + +#[test] +fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir_without_a_lockfile() + { + assert_prune_keeps_every_package_installed_by_copy( + |workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\nlockfile: false\n"), + "vs/lock.yaml", + ); +} + +#[test] +fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_modules_dir() { + assert_prune_keeps_every_package_installed_by_copy( + |workspace| append_workspace_yaml(workspace, "modulesDir: pnpm_modules\n"), + "pnpm_modules/.pnpm/lock.yaml", + ); +} + +/// The project records the shared store as its `virtualStoreDir`, and the +/// install keeps the lockfile in `node_modules/.pnpm`. +#[test] +fn store_prune_keeps_every_package_installed_by_copy_in_the_global_virtual_store() { + assert_prune_keeps_every_package_installed_by_copy( + |workspace| enable_gvs_in_workspace_yaml(workspace, ""), + "node_modules/.pnpm/lock.yaml", + ); +} + +fn assert_prune_keeps_every_package_installed_by_copy( + configure: impl FnOnce(&Path), + installed_lockfile: &str, +) { + let CommandTempCwd { + root: _root, workspace, npmrc_info, .. + } = CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { + mock_instance: _mock_instance, + store_dir, + .. + } = npmrc_info; + write_manifest(&workspace); + append_workspace_yaml(&workspace, "packageImportMethod: copy\n"); + configure(&workspace); + pacquet_at(&workspace) + .with_args(["install", "--reporter=silent"]) + .assert() + .success(); + assert!( + workspace.join(installed_lockfile).is_file(), + "the install must write {installed_lockfile}", + ); + + let store_dir = StoreDir::new(store_dir); + let keys_before = sorted_index_keys(&store_dir); + assert!(keys_before.len() > 1, "the install must index every package it fetched"); + let blob_paths = blob_paths_of_rows(&store_dir, &keys_before); + + prune(&workspace); + + assert_eq!(sorted_index_keys(&store_dir), keys_before); + let missing: Vec<_> = blob_paths + .iter() + .filter(|path| !path.exists()) + .collect(); + assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses"); +} + +/// The install keys a runtime row by its synthesized manifest as well as +/// its integrity, and the resolution lists one archive per platform. +#[test] +fn store_prune_keeps_a_runtime_installed_by_copy() { + let root = tempfile::tempdir().expect("create a temp dir"); + let workspace = prepare_workspace(&root, "packageImportMethod: copy\n"); + let mut server = mockito::Server::new(); + let fixtures = [runtime_fixture(&mut server, "node", "22.0.0", host_platform(), host_arch())]; + write_runtime_manifest(&workspace, &fixtures); + write_runtime_lockfile(&workspace, &fixtures); + pacquet_at(&workspace) + .with_args(["install", "--frozen-lockfile", "--reporter=silent"]) + .assert() + .success(); + + let store_dir = StoreDir::new(root.path().join("store")); + let keys_before = sorted_index_keys(&store_dir); + let runtime_keys: Vec = keys_before + .iter() + .filter(|key| key.ends_with("\tnode@runtime:22.0.0")) + .cloned() + .collect(); + let blob_paths = blob_paths_of_rows(&store_dir, &runtime_keys); + assert!(!blob_paths.is_empty(), "the install must index the runtime"); + + prune(&workspace); + + assert_eq!(sorted_index_keys(&store_dir), keys_before); + let missing: Vec<_> = blob_paths + .iter() + .filter(|path| !path.exists()) + .collect(); + assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed the runtime's files"); +} + +/// A lockfile written by a pnpm that uses another lockfile format, such as +/// pnpm 8. +const LOCKFILE_V6: &str = "lockfileVersion: '6.0' + +dependencies: + is-positive: + specifier: 1.0.0 + version: 1.0.0 + +packages: + + /is-positive@1.0.0: + resolution: {integrity: sha512-xxzPGZ4P2uN6rROUa5N9Z7zTX6ERuE0hs6GUOc/cKBLF2NqKc16UwqHMt3tFg4CO6EBTE5UecUasg+3jZx3Ckg==} + dev: false +"; + +#[test] +fn store_prune_skips_an_installed_lockfile_of_an_unsupported_version_with_a_warning() { + let CommandTempCwd { + root: _root, workspace, npmrc_info, .. + } = CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { mock_instance: _mock_instance, .. } = npmrc_info; + write_manifest(&workspace); + append_workspace_yaml(&workspace, "packageImportMethod: copy\n"); + pacquet_at(&workspace) + .with_args(["install", "--reporter=silent"]) + .assert() + .success(); + fs::write(workspace.join("node_modules/.pnpm/lock.yaml"), LOCKFILE_V6) + .expect("write the old lockfile"); + + let output = pacquet_at(&workspace) + .with_args(["store", "prune"]) + .output() + .expect("run store prune"); + + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("STDERR:\n{stderr}\n"); + assert!(output.status.success(), "store prune failed: {stderr}"); + let expected = format!( + "Skipped {} because it was installed with lockfileVersion 6.0, \ + which this version of pnpm does not support", + dunce::canonicalize(&workspace).expect("canonicalize the workspace").display(), + ); + let warnings: Vec<_> = stderr + .lines() + .filter(|line| line.contains("lockfileVersion")) + .collect(); + assert_eq!(warnings, [expected.as_str()]); +} + +#[test] +fn store_prune_fails_on_a_broken_installed_lockfile_and_removes_nothing() { + assert_prune_fails_on_a_broken_file_and_removes_nothing( + "node_modules/.pnpm/lock.yaml", + "packages: [", + ); +} + +#[test] +fn store_prune_fails_on_a_broken_modules_manifest_and_removes_nothing() { + assert_prune_fails_on_a_broken_file_and_removes_nothing("node_modules/.modules.yaml", "{"); +} + +fn assert_prune_fails_on_a_broken_file_and_removes_nothing(file: &str, content: &str) { + let CommandTempCwd { + root: _root, workspace, npmrc_info, .. + } = CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { + mock_instance: _mock_instance, + store_dir, + .. + } = npmrc_info; + write_manifest(&workspace); + append_workspace_yaml(&workspace, "packageImportMethod: copy\n"); + pacquet_at(&workspace) + .with_args(["install", "--reporter=silent"]) + .assert() + .success(); + fs::write(workspace.join(file), content).expect("break the file"); + let store_dir = StoreDir::new(store_dir); + let keys_before = sorted_index_keys(&store_dir); + + let output = pacquet_at(&workspace) + .with_args(["store", "prune"]) + .output() + .expect("run store prune"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "store prune must fail: {stderr}"); + assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}"); + assert_eq!(sorted_index_keys(&store_dir), keys_before); +} + +#[test] +fn store_prune_fails_on_an_undecodable_row_of_an_installed_package_and_removes_nothing() { + let CommandTempCwd { + root: _root, workspace, npmrc_info, .. + } = CommandTempCwd::init().add_mocked_registry(); + let AddMockedRegistry { + mock_instance: _mock_instance, + store_dir, + .. + } = npmrc_info; + write_manifest(&workspace); + append_workspace_yaml(&workspace, "packageImportMethod: copy\n"); + pacquet_at(&workspace) + .with_args(["install", "--reporter=silent"]) + .assert() + .success(); + let store_dir = StoreDir::new(store_dir); + let keys_before = sorted_index_keys(&store_dir); + let blob_paths = blob_paths_of_rows(&store_dir, &keys_before); + Command::new("node") + .with_args([ + "-e", + r" + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(process.argv[1]); + db.prepare(`UPDATE package_index SET data = ? WHERE key LIKE '%' || char(9) || 'is-positive@1.0.0'`) + .run(Buffer.from([0xc1])); + db.close(); + ", + ]) + .with_arg(store_dir.root().join("index.db")) + .assert() + .success(); + + let output = pacquet_at(&workspace) + .with_args(["store", "prune"]) + .output() + .expect("run store prune"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "store prune must fail: {stderr}"); + assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}"); + assert_eq!(sorted_index_keys(&store_dir), keys_before); + let missing: Vec<_> = blob_paths + .iter() + .filter(|path| !path.exists()) + .collect(); + assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses"); +} diff --git a/pnpm/crates/deps-restorer/src/install_package_by_snapshot.rs b/pnpm/crates/deps-restorer/src/install_package_by_snapshot.rs index da8d0c75e8..8284170c64 100644 --- a/pnpm/crates/deps-restorer/src/install_package_by_snapshot.rs +++ b/pnpm/crates/deps-restorer/src/install_package_by_snapshot.rs @@ -1,4 +1,4 @@ -pub use runtime::{host_platform_selector, runtime_platform_selector}; +pub use runtime::{binary_store_index_key, host_platform_selector, runtime_platform_selector}; pub use tarball_resolution::{ local_file_tarball_install_url, tarball_url_and_integrity, unverified_fetch_is_allowed, }; diff --git a/pnpm/crates/deps-restorer/src/install_package_by_snapshot/runtime.rs b/pnpm/crates/deps-restorer/src/install_package_by_snapshot/runtime.rs index c56c6fc8bf..b792d93e5b 100644 --- a/pnpm/crates/deps-restorer/src/install_package_by_snapshot/runtime.rs +++ b/pnpm/crates/deps-restorer/src/install_package_by_snapshot/runtime.rs @@ -367,6 +367,16 @@ impl BinaryArchiveFetch<'_> { .map_err(InstallPackageBySnapshotError::DownloadTarball) } } +/// The store index key of the row that a runtime install writes for +/// `binary`, which includes its synthesized manifest. +pub fn binary_store_index_key( + package_key: &PackageKey, + binary: &BinaryResolution, +) -> Result { + let manifest = synthesize_runtime_manifest_bytes(package_key, binary)?; + Ok(pnpm_tarball::ArchiveStoreProjection::Package { append_manifest: Some(&manifest) } + .store_index_key(&binary.integrity.to_string(), &package_key.pkg_id())) +} /// Serialize the synthesized runtime `package.json` to bytes. /// /// `serde_json::to_vec` writes a single-line UTF-8 blob. The bytes go diff --git a/pnpm/crates/lockfile/src/load_lockfile.rs b/pnpm/crates/lockfile/src/load_lockfile.rs index d1fdfa5942..72bd075d05 100644 --- a/pnpm/crates/lockfile/src/load_lockfile.rs +++ b/pnpm/crates/lockfile/src/load_lockfile.rs @@ -1,5 +1,6 @@ use crate::{ - EnvLockfile, Lockfile, ProjectSnapshot, extract_env_document, extract_main_document, + ComVer, EnvLockfile, Lockfile, LockfileVersion, ProjectSnapshot, extract_env_document, + extract_main_document, git_merge_file::{MERGE_CONFLICT_OURS, ParsedWantedFile, parse_wanted_file}, load_lockfile::repair_document::prepare_value_for_fix, merge_lockfile_changes, @@ -7,6 +8,7 @@ use crate::{ use derive_more::{Display, Error}; use pipe_trait::Pipe; use pnpm_diagnostics::miette::{self, Diagnostic}; +use serde::Deserialize; use serde_saphyr::MessageFormatter; use std::{ collections::HashMap, @@ -134,6 +136,26 @@ impl Lockfile { Self::load_from_path(&file_path) } + /// The `lockfileVersion` of the lockfile at `file_path` when + /// [`LockfileVersion`] rejects it, which makes every load of that file + /// fail. `None` when the file is absent or its version is accepted or + /// unreadable. + pub fn unsupported_version_at(file_path: &Path) -> Result, LoadLockfileError> { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct VersionField { + lockfile_version: ComVer, + } + let Some(content) = read_lockfile_text(file_path)? else { return Ok(None) }; + let main = extract_main_document(&content); + let Ok(VersionField { lockfile_version }) = + serde_saphyr::from_str_with_options(&main, yaml_parse_options(main.len())) + else { + return Ok(None); + }; + Ok((!LockfileVersion::<9>::is_compatible(lockfile_version)).then_some(lockfile_version)) + } + /// Load the wanted lockfile (`/pnpm-lock.yaml`) — a /// directory-addressed loader for callers that resolve into a /// directory other than the process's current one. Returns diff --git a/pnpm/crates/lockfile/src/load_lockfile/tests.rs b/pnpm/crates/lockfile/src/load_lockfile/tests.rs index ff0438109c..f22a33641f 100644 --- a/pnpm/crates/lockfile/src/load_lockfile/tests.rs +++ b/pnpm/crates/lockfile/src/load_lockfile/tests.rs @@ -628,3 +628,22 @@ fn parses_link_dep_in_injected_snapshot() { assert_eq!(c_ref, &SnapshotDepRef::Link("packages/c".to_string())); assert_eq!(c_ref.resolve(&c_name), None); } + +#[test] +fn unsupported_version_at_names_only_a_rejected_version() { + let tmp = tempdir().expect("create tempdir"); + let path = tmp.path().join(Lockfile::CURRENT_FILE_NAME); + let version_at = |content: Option<&str>| { + if let Some(content) = content { + std::fs::write(&path, content).expect("write lock.yaml"); + } + Lockfile::unsupported_version_at(&path) + .expect("read lock.yaml") + .map(|version| version.to_string()) + }; + + assert_eq!(version_at(None), None); + assert_eq!(version_at(Some("lockfileVersion: '6.0'\npackages: {}\n")), Some("6.0".to_string())); + assert_eq!(version_at(Some(MAIN_DOC)), None); + assert_eq!(version_at(Some("packages: [")), None); +} diff --git a/pnpm/crates/store-dir/src/prune.rs b/pnpm/crates/store-dir/src/prune.rs index 79571cca48..48d4f1105a 100644 --- a/pnpm/crates/store-dir/src/prune.rs +++ b/pnpm/crates/store-dir/src/prune.rs @@ -39,6 +39,9 @@ pub enum PruneError { #[display("Failed to read CAS loader references: {_0}")] #[diagnostic(code(ERR_PNPM_STORE_LOADER_REFERENCES))] LoaderReferences(#[error(source)] io::Error), + #[display("Failed to read the store files that registered projects use: {_0}")] + #[diagnostic(code(ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES))] + InUseFiles(#[error(source)] io::Error), #[diagnostic(transparent)] StoreLock(#[error(source)] StoreLockError), @@ -109,9 +112,23 @@ impl StoreDir { /// /// [#344]: https://github.com/pnpm/pacquet/issues/344 pub fn prune(&self, cache_dir: &Path, protected_dirs: &[&Path]) -> Result<(), PruneError> { + self.prune_keeping(cache_dir, protected_dirs, || Ok(HashSet::new())) + } + + /// [`Self::prune`], keeping every content-addressable file that + /// `in_use_files` returns whatever its hard-link count. `in_use_files` + /// runs while the prune holds the store lock, and its error stops the + /// prune before it removes anything. + pub fn prune_keeping( + &self, + cache_dir: &Path, + protected_dirs: &[&Path], + in_use_files: impl FnOnce() -> Result, PruneError>, + ) -> Result<(), PruneError> { let _store_lock = self.lock_for_prune().map_err(PruneError::StoreLock)?; - let references = crate::loader_references::loader_references(self) + let mut references = crate::loader_references::loader_references(self) .map_err(PruneError::LoaderReferences)?; + references.files.extend(in_use_files()?); self.prune_global_virtual_store(&references.package_roots)?; self.remove_cached_metadata(cache_dir, protected_dirs)?; let stats = crate::prune_cas::prune_cas_with_references(self, &references.files) diff --git a/pnpm/crates/store-dir/src/prune/tests.rs b/pnpm/crates/store-dir/src/prune/tests.rs index 49cc4725da..cbae6ed234 100644 --- a/pnpm/crates/store-dir/src/prune/tests.rs +++ b/pnpm/crates/store-dir/src/prune/tests.rs @@ -265,3 +265,40 @@ fn loaded_project_with_nonhidden_local_store_does_not_mark_unused_store_slots() assert!(!dead_child.exists()); } } + +#[test] +fn prune_keeping_keeps_in_use_files_and_their_rows() { + let root = tempdir().unwrap(); + let store = StoreDir::new(root.path().join("store")); + let digest = |head: &str| format!("{head}{}", "a".repeat(126)); + let in_use = store.file_path_by_hex_str(&digest("01"), ""); + let unused = store.file_path_by_hex_str(&digest("02"), ""); + let index = crate::StoreIndex::open_in(&store).unwrap(); + for (key, path, digest) in + [("in-use", &in_use, digest("01")), ("unused", &unused, digest("02"))] + { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "{}").unwrap(); + let file = crate::CafsFileInfo { digest, mode: 0o644, size: 2, checked_at: None }; + let row = crate::PackageFilesIndex { + files: std::iter::once(("package.json".to_string(), file)).collect(), + ..crate::PackageFilesIndex::default() + }; + index.set(key, &row).unwrap(); + } + drop(index); + + store + .prune_keeping(&store.root().join("cache"), &[], || { + Ok(std::iter::once(in_use.clone()).collect()) + }) + .unwrap(); + + assert!(in_use.exists()); + assert!(!unused.exists()); + let keys = crate::StoreIndex::open_in(&store) + .unwrap() + .keys() + .unwrap(); + assert_eq!(keys, ["in-use"]); +} diff --git a/pnpm11/installing/deps-installer/test/install/gitBranchLockfile.test.ts b/pnpm11/installing/deps-installer/test/install/gitBranchLockfile.test.ts index b816789c26..0f2da8b583 100644 --- a/pnpm11/installing/deps-installer/test/install/gitBranchLockfile.test.ts +++ b/pnpm11/installing/deps-installer/test/install/gitBranchLockfile.test.ts @@ -7,15 +7,17 @@ import { prepareEmpty, preparePackages } from '@pnpm/prepare' import type { ProjectManifest, ProjectRootDir } from '@pnpm/types' import { writeYamlFileSync } from 'write-yaml-file' -import { testDefaults } from '../utils/index.js' - +const gitUtils = await import('@pnpm/network.git-utils') jest.unstable_mockModule('@pnpm/network.git-utils', () => ({ + ...gitUtils, getCurrentBranch: jest.fn(), getBranchesContainingHead: jest.fn(() => Promise.resolve([])), })) const { getCurrentBranch, getBranchesContainingHead } = await import('@pnpm/network.git-utils') const { install, mutateModules } = await import('@pnpm/installing.deps-installer') +// The test store loads `@pnpm/store.controller`, which reads lockfiles through `@pnpm/network.git-utils`. +const { testDefaults } = await import('../utils/index.js') test('install with git-branch-lockfile = true', async () => { prepareEmpty() diff --git a/pnpm11/lockfile/fs/src/errors/LockfileBreakingChangeError.ts b/pnpm11/lockfile/fs/src/errors/LockfileBreakingChangeError.ts index 91f968355b..68623e0461 100644 --- a/pnpm11/lockfile/fs/src/errors/LockfileBreakingChangeError.ts +++ b/pnpm11/lockfile/fs/src/errors/LockfileBreakingChangeError.ts @@ -2,6 +2,7 @@ import { PnpmError } from '@pnpm/error' export class LockfileBreakingChangeError extends PnpmError { public filename: string + public lockfileVersion?: string constructor ( filename: string, opts?: { @@ -11,6 +12,7 @@ export class LockfileBreakingChangeError extends PnpmError { ) { super('LOCKFILE_BREAKING_CHANGE', formatMessage(filename, opts)) this.filename = filename + this.lockfileVersion = opts?.lockfileVersion } } diff --git a/pnpm11/store/commands/test/store/storePrune.ts b/pnpm11/store/commands/test/store/storePrune.ts index f501e05857..ffc862f4a6 100644 --- a/pnpm11/store/commands/test/store/storePrune.ts +++ b/pnpm11/store/commands/test/store/storePrune.ts @@ -6,8 +6,10 @@ import { assertStore } from '@pnpm/assert-store' import { STORE_VERSION } from '@pnpm/constants' import { dlx } from '@pnpm/exec.commands' import { prepare, prepareEmpty } from '@pnpm/prepare' +import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs' import { store } from '@pnpm/store.commands' -import { REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock' +import { StoreIndex } from '@pnpm/store.index' +import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock' import { rimrafSync } from '@zkochan/rimraf' import { safeExeca as execa } from 'execa' @@ -787,3 +789,237 @@ test('prune keeps the lockfile verification log', async () => { expect(fs.existsSync(verificationLog)).toBeTruthy() }) + +describe('prune keeps the packages that a registered project still uses', () => { + // The patch adds a file, so pnpm caches it as a side-effects entry of the row. + const MARKER_PATCH = [ + 'diff --git a/patched-marker.txt b/patched-marker.txt', + 'new file mode 100644', + 'index 0000000..3f2e1d4', + '--- /dev/null', + '+++ b/patched-marker.txt', + '@@ -0,0 +1 @@', + '+patched', + '', + ].join('\n') + + function prepareProject (workspaceYaml: string): void { + prepare({ + dependencies: { + '@pnpm.e2e/pkg-with-1-dep': '100.0.0', + 'is-positive': '1.0.0', + }, + }) + fs.writeFileSync('pnpm-workspace.yaml', workspaceYaml) + } + + // The store is outside the project, so the install registers the project. + async function install (): Promise { + const storeDir = path.resolve('..', 'store') + await execa('node', [pnpmBin, 'install', `--store-dir=${storeDir}`, `--cache-dir=${path.resolve('..', 'cache')}`, `--registry=${REGISTRY}`]) + return path.join(storeDir, STORE_VERSION) + } + + async function prune (storeDir: string, reporter?: (logObj: unknown) => void): Promise { + await store.handler({ + cacheDir: path.resolve('..', 'cache'), + dir: process.cwd(), + pnpmHomeDir: '', + configByUri: {}, + registriesByScope: { default: REGISTRY }, + reporter, + storeDir, + dlxCacheMaxAge: Infinity, + virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120, + }, ['prune']) + } + + function withStoreIndex (storeDir: string, read: (storeIndex: StoreIndex) => Result): Result { + const storeIndex = new StoreIndex(storeDir) + try { + return read(storeIndex) + } finally { + storeIndex.close() + } + } + + function sortedIndexKeys (storeDir: string): string[] { + return withStoreIndex(storeDir, (storeIndex) => Array.from(storeIndex.keys()).sort()) + } + + // Every store file the rows under `keys` name, with their side effects. + function blobPathsOfRows (storeDir: string, keys: string[]): string[] { + return withStoreIndex(storeDir, (storeIndex) => keys.flatMap((key) => { + const row = storeIndex.get(key) as PackageFilesIndex + return [ + ...row.files.values(), + ...Array.from(row.sideEffects?.values() ?? []).flatMap((diff) => Array.from(diff.added?.values() ?? [])), + ].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode)) + })) + } + + async function expectPruneKeepsEveryRowAndBlob (storeDir: string): Promise { + const keysBefore = sortedIndexKeys(storeDir) + expect(keysBefore.length).toBeGreaterThan(1) + const blobPaths = blobPathsOfRows(storeDir, keysBefore) + + await prune(storeDir) + + expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore) + expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([]) + } + + test('a patched dependency installed with packageImportMethod: hardlink', async () => { + // A patched package gets its own copy of the files even under `hardlink`, + // so no store file of it has a hard link outside the store. + prepareProject('packageImportMethod: hardlink\npatchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n') + fs.mkdirSync('patches') + fs.writeFileSync('patches/is-positive@1.0.0.patch', MARKER_PATCH) + const storeDir = await install() + const row = withStoreIndex(storeDir, (storeIndex) => { + const key = Array.from(storeIndex.keys()).find((key) => key.endsWith('\tis-positive@1.0.0'))! + return storeIndex.get(key) as PackageFilesIndex + }) + const sideEffects = Array.from(row.sideEffects?.entries() ?? []) + expect(sideEffects.filter(([key]) => key.includes(';patch='))).toHaveLength(1) + const blobPaths = [ + ...row.files.values(), + ...sideEffects.flatMap(([, diff]) => Array.from(diff.added?.values() ?? [])), + ].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode)) + const keysBefore = sortedIndexKeys(storeDir) + + await prune(storeDir) + + expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore) + expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([]) + }) + + test('every package installed with packageImportMethod: copy', async () => { + // No store file has a hard link outside the store, so the hard-link + // count alone marks every package unused. + prepareProject('packageImportMethod: copy\n') + const storeDir = await install() + + await expectPruneKeepsEveryRowAndBlob(storeDir) + }) + + // Without a wanted lockfile, the install writes the installed lockfile into + // the custom virtual store directory. Under a global virtual store, the + // project records the shared store as its virtualStoreDir and keeps the + // installed lockfile in node_modules/.pnpm. + test.each([ + ['modulesDir: pnpm_modules\n', 'pnpm_modules/.pnpm/lock.yaml'], + ['virtualStoreDir: vs\n', 'node_modules/.pnpm/lock.yaml'], + ['virtualStoreDir: vs\nlockfile: false\n', 'vs/lock.yaml'], + ['enableGlobalVirtualStore: true\n', 'node_modules/.pnpm/lock.yaml'], + ])('every package installed with packageImportMethod: copy and %j', async (workspaceYaml, installedLockfile) => { + prepareProject(`packageImportMethod: copy\n${workspaceYaml}`) + const storeDir = await install() + expect(fs.existsSync(installedLockfile)).toBe(true) + + await expectPruneKeepsEveryRowAndBlob(storeDir) + }) + + test('every package installed with packageImportMethod: copy and an absolute modulesDir', async () => { + prepareProject('packageImportMethod: copy\n') + const modulesDir = path.resolve('..', 'absolute_modules') + fs.appendFileSync('pnpm-workspace.yaml', `modulesDir: ${JSON.stringify(modulesDir)}\n`) + const storeDir = await install() + expect(fs.existsSync(path.join(modulesDir, '.pnpm', 'lock.yaml'))).toBe(true) + + await expectPruneKeepsEveryRowAndBlob(storeDir) + }) + + // Each project of a workspace without a shared lockfile is registered on + // its own, and the workspace-level modulesDir applies to it. + test('every package of a workspace project with its own lockfile and a workspace-level modulesDir', async () => { + prepare({ name: 'root' }) + fs.writeFileSync('pnpm-workspace.yaml', 'packages: [project]\nsharedWorkspaceLockfile: false\nmodulesDir: pnpm_modules\npackageImportMethod: copy\n') + fs.mkdirSync('project') + fs.writeFileSync('project/package.json', JSON.stringify({ + name: 'project', + dependencies: { '@pnpm.e2e/pkg-with-1-dep': '100.0.0', 'is-positive': '1.0.0' }, + })) + const storeDir = await install() + expect(fs.existsSync('project/pnpm_modules/.pnpm/lock.yaml')).toBe(true) + + await expectPruneKeepsEveryRowAndBlob(storeDir) + }) + + test('a runtime installed with packageImportMethod: copy', async () => { + // A registry tarball stands in for the runtime archive, so the install + // needs no network. The variant targets this host. + prepare({ dependencies: { node: 'runtime:22.0.0' } }) + fs.writeFileSync('pnpm-workspace.yaml', 'packageImportMethod: copy\n') + const target = { os: process.platform, cpu: process.arch } + fs.writeFileSync('pnpm-lock.yaml', JSON.stringify({ + lockfileVersion: '9.0', + settings: { autoInstallPeers: true, excludeLinksFromLockfile: false }, + importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } }, + packages: { + 'node@runtime:22.0.0': { + hasBin: true, + resolution: { + type: 'variations', + variants: [{ + targets: [target], + resolution: { + type: 'binary', + archive: 'tarball', + url: `${REGISTRY}is-positive/-/is-positive-1.0.0.tgz`, + integrity: getIntegrity('is-positive', '1.0.0'), + bin: { node: 'index.js' }, + }, + }], + }, + version: '22.0.0', + }, + }, + snapshots: { 'node@runtime:22.0.0': {} }, + })) + const storeDir = await install() + const keysBefore = sortedIndexKeys(storeDir) + const blobPaths = withStoreIndex(storeDir, (storeIndex) => keysBefore + .filter((key) => key.endsWith('\tnode@runtime:22.0.0')) + .flatMap((key) => Array.from((storeIndex.get(key) as PackageFilesIndex).files.values())) + .map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode))) + expect(blobPaths.length).toBeGreaterThan(0) + + await prune(storeDir) + + expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore) + expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([]) + }) + + test('a project installed with an unsupported lockfile version is skipped with a warning', async () => { + prepareProject('packageImportMethod: copy\n') + const storeDir = await install() + // A lockfile written by a pnpm that uses another lockfile format, such as pnpm 8. + fs.writeFileSync('node_modules/.pnpm/lock.yaml', [ + "lockfileVersion: '6.0'", + '', + 'dependencies:', + ' is-positive:', + ' specifier: 1.0.0', + ' version: 1.0.0', + '', + 'packages:', + '', + ' /is-positive@1.0.0:', + ` resolution: {integrity: ${getIntegrity('is-positive', '1.0.0')}}`, + ' dev: false', + '', + ].join('\n')) + const reporter = jest.fn() + + await prune(storeDir, reporter) + + const warnings = reporter.mock.calls + .map(([logObj]) => logObj as { level?: string, message?: string }) + .filter(({ level }) => level === 'warn') + .map(({ message }) => message) + expect(warnings).toStrictEqual([ + `Skipped ${process.cwd()} because it was installed with lockfileVersion 6.0, which this version of pnpm does not support`, + ]) + }) +}) diff --git a/pnpm11/store/controller/package.json b/pnpm11/store/controller/package.json index e3c93c217e..7094c90072 100644 --- a/pnpm11/store/controller/package.json +++ b/pnpm11/store/controller/package.json @@ -44,18 +44,25 @@ ".test": "pn pretest && cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest" }, "dependencies": { + "@pnpm/constants": "workspace:*", "@pnpm/crypto.hash": "workspace:*", "@pnpm/error": "workspace:*", "@pnpm/fetching.fetcher-base": "workspace:*", "@pnpm/hooks.types": "workspace:*", + "@pnpm/installing.modules-yaml": "workspace:*", "@pnpm/installing.package-requester": "workspace:*", + "@pnpm/lockfile.fs": "workspace:*", + "@pnpm/lockfile.types": "workspace:*", + "@pnpm/lockfile.utils": "workspace:*", "@pnpm/resolving.resolver-base": "workspace:*", "@pnpm/store.cafs": "workspace:*", "@pnpm/store.controller-types": "workspace:*", "@pnpm/store.create-cafs-store": "workspace:*", "@pnpm/store.index": "workspace:*", "@pnpm/types": "workspace:*", + "@pnpm/workspace.workspace-manifest-reader": "workspace:*", "@zkochan/rimraf": "catalog:", + "empathic": "catalog:", "is-subdir": "catalog:", "pretty-bytes": "catalog:", "ramda": "catalog:", diff --git a/pnpm11/store/controller/src/storeController/inUseFiles.ts b/pnpm11/store/controller/src/storeController/inUseFiles.ts new file mode 100644 index 0000000000..42964f9e42 --- /dev/null +++ b/pnpm11/store/controller/src/storeController/inUseFiles.ts @@ -0,0 +1,215 @@ +import path from 'node:path' + +import { LOCKFILE_VERSION, WORKSPACE_MANIFEST_FILENAME } from '@pnpm/constants' +import { isError, PnpmError } from '@pnpm/error' +import { type Modules, readModulesManifest } from '@pnpm/installing.modules-yaml' +import { readCurrentLockfile } from '@pnpm/lockfile.fs' +import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types' +import { packageIdFromSnapshot } from '@pnpm/lockfile.utils' +import { globalWarn } from '@pnpm/logger' +import { getFilePathByModeInCafs, type PackageFileInfo, type PackageFilesIndex } from '@pnpm/store.cafs' +import { gitHostedStoreIndexKey, pickStoreIndexKey, type StoreIndex, storeIndexKey } from '@pnpm/store.index' +import type { DepPath } from '@pnpm/types' +import { readWorkspaceManifestKeys } from '@pnpm/workspace.workspace-manifest-reader' +import * as find from 'empathic/find' + +import { getRegisteredProjects } from './projectRegistry.js' + +/** + * Every file of the store index rows of the packages in each registered + * project's installed lockfile, including the files of every side-effects + * entry. A package imported by copy or clone, such as a patched or built + * one, has no hard link to the store, so the prune needs this list to keep + * it. A project without an installed lockfile, a project whose lockfile + * version this pnpm does not support, and a package without a row add + * nothing. + */ +export async function getInUseFiles (storeDir: string, storeIndex: StoreIndex): Promise> { + const { keys, binaryKeys } = await registeredProjectsIndexKeys(storeDir) + for (const key of manifestIndexKeys(storeIndex, storeDir, binaryKeys)) keys.add(key) + const files = new Set() + for (const key of keys) { + const row = readIndexRow(storeIndex, storeDir, key) + if (row == null) continue + for (const { digest, mode } of rowFiles(row)) { + files.add(path.normalize(getFilePathByModeInCafs(storeDir, digest, mode))) + } + } + return files +} + +interface IndexKey { + key: string + binary: boolean +} + +async function registeredProjectsIndexKeys (storeDir: string): Promise<{ keys: Set, binaryKeys: Set }> { + const keys = new Set() + const binaryKeys = new Set() + for (const projectDir of await getRegisteredProjects(storeDir)) { + // eslint-disable-next-line no-await-in-loop -- hold one project's parsed lockfiles at a time + for (const { key, binary } of await projectIndexKeys(projectDir)) { + keys.add(key) + if (binary) binaryKeys.add(key) + } + } + return { keys, binaryKeys } +} + +async function projectIndexKeys (projectDir: string): Promise { + return (await readInstalledLockfiles(projectDir)).flatMap(lockfileIndexKeys) +} + +function lockfileIndexKeys (lockfile: LockfileObject): IndexKey[] { + return Object.entries(lockfile.packages ?? {}).flatMap(([depPath, snapshot]) => + packageIndexKeys(snapshot.resolution, packageIdFromSnapshot(depPath as DepPath, snapshot)) + ) +} + +/** + * pnpm 12 shares this store and keys the row of a runtime as + * `package-manifest\t\t`. The lockfile has + * no manifest integrity, so this scans the keys, but only when the + * lockfiles name a binary. + */ +function manifestIndexKeys (storeIndex: StoreIndex, storeDir: string, binaryKeys: Set): string[] { + if (binaryKeys.size === 0) return [] + const prefix = 'package-manifest\t' + const keys: string[] = [] + try { + for (const key of storeIndex.keys()) { + if (!key.startsWith(prefix)) continue + const separator = key.indexOf('\t', prefix.length) + if (separator !== -1 && binaryKeys.has(key.slice(separator + 1))) keys.push(key) + } + } catch (err: unknown) { + throw inUseFilesError(storeDir, err) + } + return keys +} + +function readIndexRow (storeIndex: StoreIndex, storeDir: string, key: string): PackageFilesIndex | undefined { + try { + return storeIndex.get(key) as PackageFilesIndex | undefined + } catch (err: unknown) { + throw inUseFilesError(storeDir, err) + } +} + +/** + * The lockfiles the last install wrote, which record what is installed in + * the project. + * + * The install writes the lockfile to `/.pnpm`, or to the + * `virtualStoreDir` that `/.modules.yaml` records when the + * project has no wanted lockfile. Under a global virtual store, that + * setting names the shared store. The manifest does not say which case + * applies, so the prune reads both. + */ +async function readInstalledLockfiles (projectDir: string): Promise { + const modulesDir = path.resolve(projectDir, await readConfiguredModulesDir(projectDir)) + const defaultDir = path.join(modulesDir, '.pnpm') + const modules = await readModulesManifestOfProject(modulesDir) + const installStateDirs = new Set([defaultDir, modules?.virtualStoreDir ?? defaultDir]) + const lockfiles = await Promise.all(Array.from(installStateDirs, async (installStateDir) => readInstalledLockfile(projectDir, installStateDir))) + return lockfiles.filter((lockfile) => lockfile != null) +} + +/** + * The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the + * project sets, or `node_modules`. A `modulesDir` set only on the command + * line, in the environment, in the global config, or in `packageConfigs` + * is not seen. + */ +async function readConfiguredModulesDir (projectDir: string): Promise { + try { + // `findWorkspaceDir` would also honor the workspace directory that the + // environment of the prune sets, which names no registered project's + // workspace. + const workspaceManifest = find.file(WORKSPACE_MANIFEST_FILENAME, { cwd: projectDir }) + if (workspaceManifest == null) return 'node_modules' + const workspaceDir = path.dirname(workspaceManifest) + // `PnpmSettings` does not declare `modulesDir`. + const manifest = await readWorkspaceManifestKeys(workspaceDir, MODULES_DIR_SETTING) as Record | undefined + const modulesDir = manifest?.modulesDir + return typeof modulesDir === 'string' ? modulesDir : 'node_modules' + } catch (err: unknown) { + throw inUseFilesError(projectDir, err) + } +} + +const MODULES_DIR_SETTING: ReadonlySet = new Set(['modulesDir']) + +async function readModulesManifestOfProject (modulesDir: string): Promise { + try { + return await readModulesManifest(modulesDir) + } catch (err: unknown) { + throw inUseFilesError(modulesDir, err) + } +} + +/** + * A lockfile of a version this pnpm cannot read adds nothing, like a + * missing one, so one project installed by another pnpm does not stop the + * prune. The hard-link check then decides for that project's files. + */ +async function readInstalledLockfile (projectDir: string, installStateDir: string): Promise { + try { + return await readCurrentLockfile(installStateDir, { ignoreIncompatible: false, wantedVersions: [LOCKFILE_VERSION] }) + } catch (err: unknown) { + const lockfileVersion = unsupportedLockfileVersion(err) + if (lockfileVersion != null) { + globalWarn(`Skipped ${projectDir} because it was installed with lockfileVersion ${lockfileVersion}, which this version of pnpm does not support`) + return null + } + throw inUseFilesError(installStateDir, err) + } +} + +function unsupportedLockfileVersion (err: unknown): string | undefined { + if (!isError(err) || !('code' in err) || err.code !== 'ERR_PNPM_LOCKFILE_BREAKING_CHANGE') return undefined + return (err as { lockfileVersion?: string }).lockfileVersion +} + +function inUseFilesError (location: string, err: unknown): PnpmError { + return new PnpmError('STORE_DIR_PRUNE_IN_USE_FILES', + `Failed to read the store files that registered projects use: ${location}: ${isError(err) ? err.message : String(err)}`) +} + +/** + * The lockfile does not say which key the install used for a git-hosted + * package, which has a key for its contents with and without a build, or for + * a variations resolution, whose variant depends on the installing host + * and its `supportedArchitectures`. + */ +function packageIndexKeys (resolution: LockfileResolution, pkgId: string): IndexKey[] { + if (!('type' in resolution) || resolution.type == null) { + if ('tarball' in resolution) { + return [true, false].map((built) => packageKey(pickStoreIndexKey(resolution, pkgId, { built }))) + } + return [packageKey(storeIndexKey(resolution.integrity, pkgId))] + } + switch (resolution.type) { + case 'git': + return [true, false].map((built) => packageKey(gitHostedStoreIndexKey(pkgId, { built }))) + case 'binary': + return [{ key: storeIndexKey(resolution.integrity, pkgId), binary: true }] + case 'variations': + return resolution.variants.flatMap((variant) => packageIndexKeys(variant.resolution, pkgId)) + default: + // A directory has no row, and the content of a custom one is not + // known here. + return [] + } +} + +function packageKey (key: string): IndexKey { + return { key, binary: false } +} + +function * rowFiles (row: PackageFilesIndex): Iterable { + yield * row.files.values() + for (const sideEffectsDiff of row.sideEffects?.values() ?? []) { + if (sideEffectsDiff.added != null) yield * sideEffectsDiff.added.values() + } +} diff --git a/pnpm11/store/controller/src/storeController/prune.ts b/pnpm11/store/controller/src/storeController/prune.ts index 386b6cd794..e75e6a0e6d 100644 --- a/pnpm11/store/controller/src/storeController/prune.ts +++ b/pnpm11/store/controller/src/storeController/prune.ts @@ -8,6 +8,7 @@ import type { StoreIndex } from '@pnpm/store.index' import { rimraf } from '@zkochan/rimraf' import prettyBytes from 'pretty-bytes' +import { getInUseFiles } from './inUseFiles.js' import { pruneGlobalVirtualStore } from './pruneGlobalVirtualStore.js' const BIG_ONE = BigInt(1) as unknown @@ -19,6 +20,10 @@ export interface PruneOptions { } export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, removeAlienFiles?: boolean): Promise { + // 0. Collect the files that registered projects still use, before any + // removal, so an unreadable lockfile stops the prune untouched + const inUseFiles = await getInUseFiles(storeDir, storeIndex) + // 1. First, prune the global virtual store // This must happen BEFORE pruning the CAS, because removing packages from // the virtual store will reduce hard link counts on files in the CAS @@ -29,7 +34,7 @@ export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, r await removeStaleSpillDirectories(storeDir) // 3. Prune the content-addressable store (CAS) - const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles) + const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles, inUseFiles) // 4. Clean up orphaned package index entries removeOrphanedIndexEntries(storeIndex, removedHashes) @@ -89,14 +94,16 @@ interface CafsPruneStats { fileCounter: number totalSize: number removedHashes: Set + inUseFiles: Set } -async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined): Promise> { +async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined, inUseFiles: Set): Promise> { const cafsDir = path.join(storeDir, 'files') const stats: CafsPruneStats = { fileCounter: 0, totalSize: 0, removedHashes: new Set(), + inUseFiles, } const dirs = await getSubdirsSafely(cafsDir) await Promise.all(dirs.map(async (dir) => { @@ -130,7 +137,7 @@ async function pruneCafsEntry ({ dir, fileName, subdir, removeAlienFiles, stats } return } - if (stat.nlink === 1 || stat.nlink === BIG_ONE) { + if ((stat.nlink === 1 || stat.nlink === BIG_ONE) && !stats.inUseFiles.has(filePath)) { stats.totalSize += stat.size await fs.unlink(filePath) stats.fileCounter++ diff --git a/pnpm11/store/controller/test/prune.ts b/pnpm11/store/controller/test/prune.ts index 259a5b4d98..895437eb2d 100644 --- a/pnpm11/store/controller/test/prune.ts +++ b/pnpm11/store/controller/test/prune.ts @@ -1,10 +1,12 @@ -import { promises as fs } from 'node:fs' +import { existsSync, promises as fs } from 'node:fs' import path from 'node:path' import { afterEach, expect, jest, test } from '@jest/globals' +import { getFilePathByModeInCafs } from '@pnpm/store.cafs' import { StoreIndex } from '@pnpm/store.index' import { temporaryDirectory } from 'tempy' +import { registerProject } from '../src/storeController/projectRegistry.js' import { prune } from '../src/storeController/prune.js' const temporaryDirectories: string[] = [] @@ -124,6 +126,93 @@ test('prune propagates unexpected spill directory errors', async () => { await expect(prune(options)).rejects.toBe(error) }) +test.each([ + ['installed lockfile', path.join('node_modules', '.pnpm', 'lock.yaml'), 'packages: ['], + ['modules manifest', path.join('node_modules', '.modules.yaml'), '{'], +])('prune fails before it removes anything when a registered project has a broken %s', async (_name, brokenFile, content) => { + const options = createPruneOptions() + const projectDir = path.join(options.cacheDir, '..', 'project') + await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true }) + await fs.writeFile(path.join(projectDir, brokenFile), content) + await registerProject(options.storeDir, projectDir) + const unusedFile = path.join(options.storeDir, 'files', '00', 'unused') + await fs.mkdir(path.dirname(unusedFile), { recursive: true }) + await fs.writeFile(unusedFile, 'unused') + + await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' }) + expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused') +}) + +test('prune fails before it removes anything when the store index cannot be read', async () => { + const options = createPruneOptions() + const projectDir = path.join(options.cacheDir, '..', 'project') + await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true }) + await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), [ + "lockfileVersion: '9.0'", + 'importers:', + " '.': {}", + 'packages:', + ' is-positive@1.0.0:', + ' resolution: {integrity: sha512-AAAA}', + 'snapshots:', + ' is-positive@1.0.0: {}', + '', + ].join('\n')) + await registerProject(options.storeDir, projectDir) + const unusedFile = path.join(options.storeDir, 'files', '00', 'unused') + await fs.mkdir(path.dirname(unusedFile), { recursive: true }) + await fs.writeFile(unusedFile, 'unused') + jest.spyOn(options.storeIndex, 'get').mockImplementation(() => { + throw new Error('database disk image is malformed') + }) + + await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' }) + expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused') +}) + +test('prune keeps a runtime whose index row is keyed with its manifest integrity', async () => { + // pnpm 12 shares this store and keys a runtime row as + // `package-manifest\t\t\t`. + const options = createPruneOptions() + const projectDir = path.join(options.cacheDir, '..', 'project') + await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true }) + const integrity = 'sha512-BBBB' + await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), JSON.stringify({ + lockfileVersion: '9.0', + importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } }, + packages: { + 'node@runtime:22.0.0': { + resolution: { + type: 'variations', + variants: [{ + targets: [{ os: 'linux', cpu: 'x64' }], + resolution: { type: 'binary', archive: 'tarball', url: 'https://example.com/node.tgz', integrity, bin: { node: 'bin/node' } }, + }], + }, + version: '22.0.0', + }, + }, + snapshots: { 'node@runtime:22.0.0': {} }, + })) + await registerProject(options.storeDir, projectDir) + const files = new Map([ + ['package.json', { digest: 'aa'.repeat(32), mode: 0o644, size: 2 }], + ['bin/node', { digest: 'bb'.repeat(32), mode: 0o755, size: 2 }], + ]) + const blobPaths = Array.from(files.values(), ({ digest, mode }) => getFilePathByModeInCafs(options.storeDir, digest, mode)) + await Promise.all(blobPaths.map(async (blobPath) => { + await fs.mkdir(path.dirname(blobPath), { recursive: true }) + await fs.writeFile(blobPath, '{}') + })) + const key = `package-manifest\tsha512-CCCC\t${integrity}\tnode@runtime:22.0.0` + options.storeIndex.set(key, { algo: 'sha512', files }) + + await prune(options) + + expect(options.storeIndex.has(key)).toBe(true) + expect(blobPaths.filter((blobPath) => !existsSync(blobPath))).toStrictEqual([]) +}) + function createPruneOptions () { const directory = temporaryDirectory() temporaryDirectories.push(directory) diff --git a/pnpm11/store/controller/tsconfig.json b/pnpm11/store/controller/tsconfig.json index aae4f656fd..27490a785d 100644 --- a/pnpm11/store/controller/tsconfig.json +++ b/pnpm11/store/controller/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../../__utils__/prepare" }, + { + "path": "../../core/constants" + }, { "path": "../../core/error" }, @@ -33,15 +36,30 @@ { "path": "../../installing/client" }, + { + "path": "../../installing/modules-yaml" + }, { "path": "../../installing/package-requester" }, + { + "path": "../../lockfile/fs" + }, + { + "path": "../../lockfile/types" + }, + { + "path": "../../lockfile/utils" + }, { "path": "../../resolving/resolver-base" }, { "path": "../../worker" }, + { + "path": "../../workspace/workspace-manifest-reader" + }, { "path": "../cafs" },