fix(store): keep packages that a project still uses on store prune (#17327)

The CAS prune removed a store file whose hard-link count was 1. A
package imported with clone or copy leaves its store files at one link
while a project still uses it. That is every patched or built package,
every package under packageImportMethod: copy, and every package of a
macOS clone. So the prune removed them, and the next install downloaded
them again.

The prune now also keeps every file that a registered project's
installed lockfile references:

- For each registered project it reads <modulesDir>/.pnpm/lock.yaml and
  the lock.yaml in the virtualStoreDir recorded in
  <modulesDir>/.modules.yaml. modulesDir comes from the nearest
  pnpm-workspace.yaml at or above the registered directory.
- Each package's store index keys come from the install's own key
  functions: both keys of a git-hosted package, the manifest-keyed and
  plain keys of a runtime, and every variant of a variations
  resolution.
- Every file of each row is kept, including each side-effects entry.
- A project with no installed lockfile and a package with no row add
  nothing, so the hard-link check still decides for them.
- A lockfile of an unsupported version is skipped with a warning. A
  lockfile, .modules.yaml, pnpm-workspace.yaml, or store index row that
  cannot be read fails the prune with
  ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES. Registry errors keep their own
  codes.

In pnpm 12, StoreDir::prune_keeping runs a caller's collector under the
prune lock, and the CLI supplies the collector because store-dir does
not depend on the lockfile crate. pnpm 11 collects before any removal.

pnpm 11 and pnpm 12 share one store. pnpm 12 keys a runtime row with
its synthesized manifest, so the pnpm 11 collector also keeps
package-manifest keys whose tail it references. It scans the index keys
only when a lockfile holds a runtime.

A modulesDir set only on the command line, in the environment, in the
global config, or in packageConfigs is not seen.

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-10-26 11:34:41 +01:00
1 parent dbfa9710e5
commit a582bb1b71
21 files changed
+1346 -16

No files matched your search

@@ -0,0 +1,8 @@
---
"@pnpm/lockfile.fs": patch
"@pnpm/store.controller": patch
"pnpm": patch
"pacquet": patch
---
`pnpm store prune` now keeps the packages that a project still uses when pnpm copied or cloned them into `node_modules`. These include patched packages, built packages, and every package installed with `packageImportMethod: copy`. Before, the prune removed them, and the next install downloaded them again [#7192](https://github.com/pnpm/pnpm/issues/7192).
+21
View File
@@ -10173,6 +10173,9 @@ importers:
pnpm11/store/controller:
dependencies:
'@pnpm/constants':
specifier: workspace:*
version: link:../../core/constants
'@pnpm/crypto.hash':
specifier: workspace:*
version: link:../../crypto/hash
@@ -10185,9 +10188,21 @@ importers:
'@pnpm/hooks.types':
specifier: workspace:*
version: link:../../hooks/types
'@pnpm/installing.modules-yaml':
specifier: workspace:*
version: link:../../installing/modules-yaml
'@pnpm/installing.package-requester':
specifier: workspace:*
version: link:../../installing/package-requester
'@pnpm/lockfile.fs':
specifier: workspace:*
version: link:../../lockfile/fs
'@pnpm/lockfile.types':
specifier: workspace:*
version: link:../../lockfile/types
'@pnpm/lockfile.utils':
specifier: workspace:*
version: link:../../lockfile/utils
'@pnpm/resolving.resolver-base':
specifier: workspace:*
version: link:../../resolving/resolver-base
@@ -10209,9 +10224,15 @@ importers:
'@pnpm/worker':
specifier: workspace:^
version: link:../../worker
'@pnpm/workspace.workspace-manifest-reader':
specifier: workspace:*
version: link:../../workspace/workspace-manifest-reader
'@zkochan/rimraf':
specifier: 'catalog:'
version: 4.0.0
empathic:
specifier: 'catalog:'
version: 2.1.0
is-subdir:
specifier: 'catalog:'
version: 2.0.1
+6 -1
View File
@@ -7,6 +7,7 @@
//! any project.
mod add;
mod in_use_files;
mod status;
use crate::{
@@ -80,7 +81,11 @@ fn prune(config: &Config) -> miette::Result<()> {
.into_iter()
.flatten()
.collect();
config.store_dir.prune(&config.cache_dir, &protected_dirs).wrap_err("pruning store")?;
config.store_dir
.prune_keeping(&config.cache_dir, &protected_dirs, || {
in_use_files::in_use_files(&config.store_dir)
})
.wrap_err("pruning store")?;
remove_orphaned_private_engine_installs(config)?;
Ok(())
}
@@ -0,0 +1,178 @@
//! The store files of the packages that registered projects installed.
//!
//! A package imported by copy or clone, such as a patched or built one, has
//! no hard link to the store, so `store prune` needs this list to keep it.
use pnpm_config::workspace_root_or;
use pnpm_deps_restorer::{binary_store_index_key, store_index_key_for_resolution};
use pnpm_fs::lexical_normalize;
use pnpm_lockfile::{Lockfile, LockfileResolution, PackageKey};
use pnpm_modules_yaml::{Host, read_modules_layout};
use pnpm_store_dir::{
CafsFileInfo, PackageFilesIndex, PruneError, StoreDir, StoreIndex, StoreIndexError,
decode_package_files_index, get_registered_projects,
};
use pnpm_workspace::read_workspace_manifest_document;
use serde::Deserialize;
use std::{
collections::HashSet,
io, iter,
path::{Path, PathBuf},
};
/// Every file of the store index rows of the packages in each registered
/// project's current lockfile, including the files of every side-effects
/// entry. A project without a current lockfile, a project whose lockfile
/// version this pnpm does not support, and a package without a row add
/// nothing. A registry error keeps its own code, and any other
/// failure is [`PruneError::InUseFiles`].
pub(super) fn in_use_files(store_dir: &StoreDir) -> Result<HashSet<PathBuf>, PruneError> {
if !StoreIndex::exists(store_dir.root()) {
return Ok(HashSet::new());
}
let mut keys = HashSet::new();
for project in get_registered_projects(store_dir).map_err(PruneError::ListProjects)? {
// The registry joins a relative link target without normalizing it.
let project = lexical_normalize(&project);
for lockfile in load_installed_lockfiles(&project).map_err(PruneError::InUseFiles)? {
keys.extend(package_index_keys(&lockfile).map_err(PruneError::InUseFiles)?);
}
}
if keys.is_empty() {
return Ok(HashSet::new());
}
let keys: Vec<String> = keys.into_iter().collect();
let rows = read_rows(store_dir, &keys)
.map_err(|error| PruneError::InUseFiles(io::Error::other(error)))?;
Ok(rows
.iter()
.flat_map(row_files)
.filter_map(|file| store_dir.cas_file_path_by_mode(&file.digest, file.mode))
.collect())
}
/// Fails on an undecodable row, which [`StoreIndex::get_many`] skips,
/// because skipping it would let the prune remove the row's files.
fn read_rows(
store_dir: &StoreDir,
keys: &[String],
) -> Result<Vec<PackageFilesIndex>, StoreIndexError> {
StoreIndex::open_readonly_in(store_dir)?
.get_many_raw(keys)?
.iter()
.map(|(_, bytes)| decode_package_files_index(bytes))
.collect()
}
/// The lockfiles the last install wrote, which record what is installed in
/// the project.
///
/// The install writes the lockfile to `<modulesDir>/.pnpm`, or to the
/// `virtualStoreDir` that `<modulesDir>/.modules.yaml` records when the
/// project sets one. Under a global virtual store, that setting names the
/// shared store and the lockfile stays in `<modulesDir>/.pnpm`. The
/// manifest does not say which case applies, so the prune reads both.
fn load_installed_lockfiles(project: &Path) -> io::Result<Vec<Lockfile>> {
let modules_dir = project.join(configured_modules_dir(project)?);
let default_dir = modules_dir.join(".pnpm");
let virtual_store_dir = read_modules_layout::<Host>(&modules_dir)
.map_err(io::Error::other)?
.map(|layout| PathBuf::from(layout.virtual_store_dir))
.filter(|dir| *dir != default_dir);
let mut lockfiles = Vec::new();
for install_state_dir in iter::once(default_dir).chain(virtual_store_dir) {
lockfiles.extend(load_installed_lockfile(project, &install_state_dir)?);
}
Ok(lockfiles)
}
/// The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the
/// project sets, or `node_modules`. A `modulesDir` set only on the command
/// line, in the environment, in the global config, or in `packageConfigs`
/// is not seen.
fn configured_modules_dir(project: &Path) -> io::Result<String> {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct ModulesDirSetting {
modules_dir: Option<String>,
}
let Some(document) =
read_workspace_manifest_document(&workspace_root_or(project)).map_err(io::Error::other)?
else {
return Ok("node_modules".to_string());
};
let setting: Option<ModulesDirSetting> = serde_saphyr::from_str(&document.text)
.map_err(|error| io::Error::other(format!("{}: {error}", document.path.display())))?;
Ok(setting
.and_then(|setting| setting.modules_dir)
.unwrap_or_else(|| "node_modules".to_string()))
}
/// A lockfile of a version this pnpm cannot read adds nothing, like a
/// missing one, so one project installed by another pnpm does not stop the
/// prune. The hard-link check then decides for that project's files.
fn load_installed_lockfile(
project: &Path,
install_state_dir: &Path,
) -> io::Result<Option<Lockfile>> {
let error = match Lockfile::load_current_from_install_state_dir(install_state_dir) {
Ok(lockfile) => return Ok(lockfile),
Err(error) => error,
};
let lockfile_path = install_state_dir.join(Lockfile::CURRENT_FILE_NAME);
if let Ok(Some(version)) = Lockfile::unsupported_version_at(&lockfile_path) {
pnpm_diagnostics::eprintln!(
"Skipped {} because it was installed with lockfileVersion {version}, \
which this version of pnpm does not support",
project.display(),
);
return Ok(None);
}
Err(io::Error::other(format!("{}: {error}", install_state_dir.display())))
}
fn package_index_keys(lockfile: &Lockfile) -> io::Result<Vec<String>> {
let mut keys = Vec::new();
for (key, metadata) in lockfile.packages.iter().flatten() {
add_resolution_index_keys(&mut keys, key, &metadata.resolution)?;
}
Ok(keys)
}
/// The lockfile does not say which key the install used for:
/// - a git-hosted package, which has a key for its built and its unbuilt
/// contents;
/// - a `Variations` resolution, whose variant depends on the installing
/// host and its `supportedArchitectures`;
/// - a runtime, whose row an older install keyed without its manifest.
fn add_resolution_index_keys(
keys: &mut Vec<String>,
key: &PackageKey,
resolution: &LockfileResolution,
) -> io::Result<()> {
if let LockfileResolution::Variations(variations) = resolution {
for variant in &variations.variants {
add_resolution_index_keys(keys, key, &variant.resolution)?;
}
return Ok(());
}
let pkg_id = key.pkg_id();
keys.extend(
[true, false]
.into_iter()
.filter_map(|built| store_index_key_for_resolution(resolution, &pkg_id, built)),
);
if let LockfileResolution::Binary(binary) = resolution {
keys.push(binary_store_index_key(key, binary).map_err(io::Error::other)?);
}
Ok(())
}
fn row_files(row: &PackageFilesIndex) -> impl Iterator<Item = &CafsFileInfo> {
let side_effects_files = row.side_effects
.iter()
.flat_map(|side_effects| side_effects.values())
.filter_map(|diff| diff.added.as_ref())
.flat_map(|added| added.values());
row.files.values().chain(side_effects_files)
}
@@ -13,7 +13,7 @@ use std::{
};
use tempfile::TempDir;
struct RuntimeFixture {
pub(crate) struct RuntimeFixture {
name: &'static str,
version: &'static str,
archive_mock: mockito::Mock,
@@ -963,7 +963,7 @@ fn assert_runtime_bad_integrity(name: &'static str, version: &'static str) {
assert!(stderr.contains("Integrity check failed"), "stderr was:\n{stderr}");
}
fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
pub(crate) fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
let workspace = root.path().join("workspace");
fs::create_dir(&workspace).unwrap();
fs::write(
@@ -976,7 +976,7 @@ fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
workspace
}
fn runtime_fixture(
pub(crate) fn runtime_fixture(
server: &mut mockito::Server,
name: &'static str,
version: &'static str,
@@ -1027,7 +1027,7 @@ fn runtime_fixture(
}
}
fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
pub(crate) fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
let dependencies = fixtures
.iter()
.map(|fixture| {
@@ -1038,7 +1038,7 @@ fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
.unwrap();
}
fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) {
pub(crate) fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) {
write_runtime_lockfile_for_group(workspace, fixtures, "dependencies");
}
+1
View File
@@ -192,6 +192,7 @@ mod store;
mod store_pkg_content_check;
mod store_placement;
mod store_project_registry;
mod store_prune_in_use;
mod sync_injected_deps_after_scripts;
mod tar_registry;
mod tarball_url_dependency;
@@ -0,0 +1,436 @@
//! `pnpm store prune` must keep the store files of every package a
//! registered project installed, including the packages that pnpm copied
//! into `node_modules`, which hold no hard link to the store.
use crate::{
_utils::enable_gvs_in_workspace_yaml,
install_runtimes::{
prepare_workspace, runtime_fixture, write_runtime_lockfile, write_runtime_manifest,
},
patch::MARKER_PATCH,
};
use assert_cmd::prelude::*;
use command_extra::CommandExtra;
use pnpm_graph_hasher::{host_arch, host_platform};
use pnpm_store_dir::{PackageFilesIndex, StoreDir, StoreIndex};
use pnpm_testing_utils::{
bin::{AddMockedRegistry, CommandTempCwd},
command_env::CommandTestExt,
};
use pretty_assertions::assert_eq;
use std::{
fs,
path::{Path, PathBuf},
process::Command,
};
fn pacquet_at(workspace: &Path) -> Command {
Command::cargo_bin("pnpm")
.expect("find the pnpm binary")
.with_current_dir(workspace)
.without_ambient_pnpm_config()
}
fn write_manifest(workspace: &Path) {
fs::write(
workspace.join("package.json"),
serde_json::json!({
"dependencies": {
"@pnpm.e2e/pkg-with-1-dep": "100.0.0",
"is-positive": "1.0.0",
},
})
.to_string(),
)
.expect("write package.json");
}
fn append_workspace_yaml(workspace: &Path, text: &str) {
let path = workspace.join("pnpm-workspace.yaml");
let mut yaml = fs::read_to_string(&path).expect("read pnpm-workspace.yaml");
if !yaml.ends_with('\n') {
yaml.push('\n');
}
yaml.push_str(text);
fs::write(&path, yaml).expect("write pnpm-workspace.yaml");
}
fn sorted_index_keys(store_dir: &StoreDir) -> Vec<String> {
let mut keys = StoreIndex::open_readonly_in(store_dir)
.expect("open the store index")
.keys()
.expect("read the store index keys");
keys.sort();
keys
}
fn is_positive_row(store_dir: &StoreDir) -> PackageFilesIndex {
let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index");
let key = index
.keys()
.expect("read the store index keys")
.into_iter()
.find(|key| key.ends_with("\tis-positive@1.0.0"))
.expect("a store index row for is-positive@1.0.0");
index
.get(&key)
.expect("read the store index row")
.expect("the row is present")
}
/// Every store file the row names: its base files and the files each
/// side-effects entry adds.
fn row_blob_paths(store_dir: &StoreDir, row: &PackageFilesIndex) -> Vec<PathBuf> {
let side_effects_files = row.side_effects
.iter()
.flat_map(|side_effects| side_effects.values())
.filter_map(|diff| diff.added.as_ref())
.flat_map(|added| added.values());
row.files
.values()
.chain(side_effects_files)
.map(|info| {
store_dir
.cas_file_path_by_mode(&info.digest, info.mode)
.expect("an index row holds hex digests")
})
.collect()
}
/// Every store file the rows under `keys` name.
fn blob_paths_of_rows(store_dir: &StoreDir, keys: &[String]) -> Vec<PathBuf> {
let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index");
keys.iter()
.flat_map(|key| {
let row = index
.get(key)
.expect("read the store index row")
.expect("the row is present");
row_blob_paths(store_dir, &row)
})
.collect()
}
fn prune(workspace: &Path) {
let output = pacquet_at(workspace)
.with_args(["store", "prune"])
.output()
.expect("run store prune");
let stderr = String::from_utf8_lossy(&output.stderr);
eprintln!("STDERR:\n{stderr}\n");
assert!(output.status.success(), "store prune failed: {stderr}");
}
/// A patched package gets its own copy of the files even under `hardlink`,
/// so no store file of it has a hard link outside the store. `hardlink`
/// keeps every other package linked on each platform, which `auto` does
/// not do on a copy-on-write filesystem.
#[test]
fn store_prune_keeps_the_files_of_a_patched_dependency() {
let CommandTempCwd {
root: _root, workspace, npmrc_info, ..
} = CommandTempCwd::init().add_mocked_registry();
let AddMockedRegistry {
mock_instance: _mock_instance,
store_dir,
..
} = npmrc_info;
write_manifest(&workspace);
fs::create_dir_all(workspace.join("patches")).expect("create the patches dir");
fs::write(workspace.join("patches/is-positive@1.0.0.patch"), MARKER_PATCH)
.expect("write the patch file");
append_workspace_yaml(
&workspace,
concat!(
"packageImportMethod: hardlink\n",
"patchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n",
),
);
pacquet_at(&workspace)
.with_args(["install", "--reporter=silent"])
.assert()
.success();
let store_dir = StoreDir::new(store_dir);
let row = is_positive_row(&store_dir);
let patched_entries = row.side_effects
.as_ref()
.map_or(0, |side_effects| {
side_effects
.keys()
.filter(|key| key.contains(";patch="))
.count()
});
assert_eq!(patched_entries, 1, "the patch must be cached as a side-effects entry");
let blob_paths = row_blob_paths(&store_dir, &row);
let keys_before = sorted_index_keys(&store_dir);
prune(&workspace);
assert_eq!(sorted_index_keys(&store_dir), keys_before);
let missing: Vec<_> = blob_paths
.iter()
.filter(|path| !path.exists())
.collect();
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
}
/// With `packageImportMethod: copy` no store file has a hard link outside
/// the store, so the hard-link count alone marks every package unused.
#[test]
fn store_prune_keeps_every_package_installed_by_copy() {
assert_prune_keeps_every_package_installed_by_copy(|_| {}, "node_modules/.pnpm/lock.yaml");
}
#[test]
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir() {
assert_prune_keeps_every_package_installed_by_copy(
|workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\n"),
"vs/lock.yaml",
);
}
#[test]
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir_without_a_lockfile()
{
assert_prune_keeps_every_package_installed_by_copy(
|workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\nlockfile: false\n"),
"vs/lock.yaml",
);
}
#[test]
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_modules_dir() {
assert_prune_keeps_every_package_installed_by_copy(
|workspace| append_workspace_yaml(workspace, "modulesDir: pnpm_modules\n"),
"pnpm_modules/.pnpm/lock.yaml",
);
}
/// The project records the shared store as its `virtualStoreDir`, and the
/// install keeps the lockfile in `node_modules/.pnpm`.
#[test]
fn store_prune_keeps_every_package_installed_by_copy_in_the_global_virtual_store() {
assert_prune_keeps_every_package_installed_by_copy(
|workspace| enable_gvs_in_workspace_yaml(workspace, ""),
"node_modules/.pnpm/lock.yaml",
);
}
fn assert_prune_keeps_every_package_installed_by_copy(
configure: impl FnOnce(&Path),
installed_lockfile: &str,
) {
let CommandTempCwd {
root: _root, workspace, npmrc_info, ..
} = CommandTempCwd::init().add_mocked_registry();
let AddMockedRegistry {
mock_instance: _mock_instance,
store_dir,
..
} = npmrc_info;
write_manifest(&workspace);
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
configure(&workspace);
pacquet_at(&workspace)
.with_args(["install", "--reporter=silent"])
.assert()
.success();
assert!(
workspace.join(installed_lockfile).is_file(),
"the install must write {installed_lockfile}",
);
let store_dir = StoreDir::new(store_dir);
let keys_before = sorted_index_keys(&store_dir);
assert!(keys_before.len() > 1, "the install must index every package it fetched");
let blob_paths = blob_paths_of_rows(&store_dir, &keys_before);
prune(&workspace);
assert_eq!(sorted_index_keys(&store_dir), keys_before);
let missing: Vec<_> = blob_paths
.iter()
.filter(|path| !path.exists())
.collect();
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
}
/// The install keys a runtime row by its synthesized manifest as well as
/// its integrity, and the resolution lists one archive per platform.
#[test]
fn store_prune_keeps_a_runtime_installed_by_copy() {
let root = tempfile::tempdir().expect("create a temp dir");
let workspace = prepare_workspace(&root, "packageImportMethod: copy\n");
let mut server = mockito::Server::new();
let fixtures = [runtime_fixture(&mut server, "node", "22.0.0", host_platform(), host_arch())];
write_runtime_manifest(&workspace, &fixtures);
write_runtime_lockfile(&workspace, &fixtures);
pacquet_at(&workspace)
.with_args(["install", "--frozen-lockfile", "--reporter=silent"])
.assert()
.success();
let store_dir = StoreDir::new(root.path().join("store"));
let keys_before = sorted_index_keys(&store_dir);
let runtime_keys: Vec<String> = keys_before
.iter()
.filter(|key| key.ends_with("\tnode@runtime:22.0.0"))
.cloned()
.collect();
let blob_paths = blob_paths_of_rows(&store_dir, &runtime_keys);
assert!(!blob_paths.is_empty(), "the install must index the runtime");
prune(&workspace);
assert_eq!(sorted_index_keys(&store_dir), keys_before);
let missing: Vec<_> = blob_paths
.iter()
.filter(|path| !path.exists())
.collect();
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed the runtime's files");
}
/// A lockfile written by a pnpm that uses another lockfile format, such as
/// pnpm 8.
const LOCKFILE_V6: &str = "lockfileVersion: '6.0'
dependencies:
is-positive:
specifier: 1.0.0
version: 1.0.0
packages:
/is-positive@1.0.0:
resolution: {integrity: sha512-xxzPGZ4P2uN6rROUa5N9Z7zTX6ERuE0hs6GUOc/cKBLF2NqKc16UwqHMt3tFg4CO6EBTE5UecUasg+3jZx3Ckg==}
dev: false
";
#[test]
fn store_prune_skips_an_installed_lockfile_of_an_unsupported_version_with_a_warning() {
let CommandTempCwd {
root: _root, workspace, npmrc_info, ..
} = CommandTempCwd::init().add_mocked_registry();
let AddMockedRegistry { mock_instance: _mock_instance, .. } = npmrc_info;
write_manifest(&workspace);
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
pacquet_at(&workspace)
.with_args(["install", "--reporter=silent"])
.assert()
.success();
fs::write(workspace.join("node_modules/.pnpm/lock.yaml"), LOCKFILE_V6)
.expect("write the old lockfile");
let output = pacquet_at(&workspace)
.with_args(["store", "prune"])
.output()
.expect("run store prune");
let stderr = String::from_utf8_lossy(&output.stderr);
eprintln!("STDERR:\n{stderr}\n");
assert!(output.status.success(), "store prune failed: {stderr}");
let expected = format!(
"Skipped {} because it was installed with lockfileVersion 6.0, \
which this version of pnpm does not support",
dunce::canonicalize(&workspace).expect("canonicalize the workspace").display(),
);
let warnings: Vec<_> = stderr
.lines()
.filter(|line| line.contains("lockfileVersion"))
.collect();
assert_eq!(warnings, [expected.as_str()]);
}
#[test]
fn store_prune_fails_on_a_broken_installed_lockfile_and_removes_nothing() {
assert_prune_fails_on_a_broken_file_and_removes_nothing(
"node_modules/.pnpm/lock.yaml",
"packages: [",
);
}
#[test]
fn store_prune_fails_on_a_broken_modules_manifest_and_removes_nothing() {
assert_prune_fails_on_a_broken_file_and_removes_nothing("node_modules/.modules.yaml", "{");
}
fn assert_prune_fails_on_a_broken_file_and_removes_nothing(file: &str, content: &str) {
let CommandTempCwd {
root: _root, workspace, npmrc_info, ..
} = CommandTempCwd::init().add_mocked_registry();
let AddMockedRegistry {
mock_instance: _mock_instance,
store_dir,
..
} = npmrc_info;
write_manifest(&workspace);
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
pacquet_at(&workspace)
.with_args(["install", "--reporter=silent"])
.assert()
.success();
fs::write(workspace.join(file), content).expect("break the file");
let store_dir = StoreDir::new(store_dir);
let keys_before = sorted_index_keys(&store_dir);
let output = pacquet_at(&workspace)
.with_args(["store", "prune"])
.output()
.expect("run store prune");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!output.status.success(), "store prune must fail: {stderr}");
assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}");
assert_eq!(sorted_index_keys(&store_dir), keys_before);
}
#[test]
fn store_prune_fails_on_an_undecodable_row_of_an_installed_package_and_removes_nothing() {
let CommandTempCwd {
root: _root, workspace, npmrc_info, ..
} = CommandTempCwd::init().add_mocked_registry();
let AddMockedRegistry {
mock_instance: _mock_instance,
store_dir,
..
} = npmrc_info;
write_manifest(&workspace);
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
pacquet_at(&workspace)
.with_args(["install", "--reporter=silent"])
.assert()
.success();
let store_dir = StoreDir::new(store_dir);
let keys_before = sorted_index_keys(&store_dir);
let blob_paths = blob_paths_of_rows(&store_dir, &keys_before);
Command::new("node")
.with_args([
"-e",
r"
const { DatabaseSync } = require('node:sqlite');
const db = new DatabaseSync(process.argv[1]);
db.prepare(`UPDATE package_index SET data = ? WHERE key LIKE '%' || char(9) || 'is-positive@1.0.0'`)
.run(Buffer.from([0xc1]));
db.close();
",
])
.with_arg(store_dir.root().join("index.db"))
.assert()
.success();
let output = pacquet_at(&workspace)
.with_args(["store", "prune"])
.output()
.expect("run store prune");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!output.status.success(), "store prune must fail: {stderr}");
assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}");
assert_eq!(sorted_index_keys(&store_dir), keys_before);
let missing: Vec<_> = blob_paths
.iter()
.filter(|path| !path.exists())
.collect();
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
}
@@ -1,4 +1,4 @@
pub use runtime::{host_platform_selector, runtime_platform_selector};
pub use runtime::{binary_store_index_key, host_platform_selector, runtime_platform_selector};
pub use tarball_resolution::{
local_file_tarball_install_url, tarball_url_and_integrity, unverified_fetch_is_allowed,
};
@@ -367,6 +367,16 @@ impl BinaryArchiveFetch<'_> {
.map_err(InstallPackageBySnapshotError::DownloadTarball)
}
}
/// The store index key of the row that a runtime install writes for
/// `binary`, which includes its synthesized manifest.
pub fn binary_store_index_key(
package_key: &PackageKey,
binary: &BinaryResolution,
) -> Result<String, InstallPackageBySnapshotError> {
let manifest = synthesize_runtime_manifest_bytes(package_key, binary)?;
Ok(pnpm_tarball::ArchiveStoreProjection::Package { append_manifest: Some(&manifest) }
.store_index_key(&binary.integrity.to_string(), &package_key.pkg_id()))
}
/// Serialize the synthesized runtime `package.json` to bytes.
///
/// `serde_json::to_vec` writes a single-line UTF-8 blob. The bytes go
+23 -1
View File
@@ -1,5 +1,6 @@
use crate::{
EnvLockfile, Lockfile, ProjectSnapshot, extract_env_document, extract_main_document,
ComVer, EnvLockfile, Lockfile, LockfileVersion, ProjectSnapshot, extract_env_document,
extract_main_document,
git_merge_file::{MERGE_CONFLICT_OURS, ParsedWantedFile, parse_wanted_file},
load_lockfile::repair_document::prepare_value_for_fix,
merge_lockfile_changes,
@@ -7,6 +8,7 @@ use crate::{
use derive_more::{Display, Error};
use pipe_trait::Pipe;
use pnpm_diagnostics::miette::{self, Diagnostic};
use serde::Deserialize;
use serde_saphyr::MessageFormatter;
use std::{
collections::HashMap,
@@ -134,6 +136,26 @@ impl Lockfile {
Self::load_from_path(&file_path)
}
/// The `lockfileVersion` of the lockfile at `file_path` when
/// [`LockfileVersion`] rejects it, which makes every load of that file
/// fail. `None` when the file is absent or its version is accepted or
/// unreadable.
pub fn unsupported_version_at(file_path: &Path) -> Result<Option<ComVer>, LoadLockfileError> {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct VersionField {
lockfile_version: ComVer,
}
let Some(content) = read_lockfile_text(file_path)? else { return Ok(None) };
let main = extract_main_document(&content);
let Ok(VersionField { lockfile_version }) =
serde_saphyr::from_str_with_options(&main, yaml_parse_options(main.len()))
else {
return Ok(None);
};
Ok((!LockfileVersion::<9>::is_compatible(lockfile_version)).then_some(lockfile_version))
}
/// Load the wanted lockfile (`<dir>/pnpm-lock.yaml`) — a
/// directory-addressed loader for callers that resolve into a
/// directory other than the process's current one. Returns
@@ -628,3 +628,22 @@ fn parses_link_dep_in_injected_snapshot() {
assert_eq!(c_ref, &SnapshotDepRef::Link("packages/c".to_string()));
assert_eq!(c_ref.resolve(&c_name), None);
}
#[test]
fn unsupported_version_at_names_only_a_rejected_version() {
let tmp = tempdir().expect("create tempdir");
let path = tmp.path().join(Lockfile::CURRENT_FILE_NAME);
let version_at = |content: Option<&str>| {
if let Some(content) = content {
std::fs::write(&path, content).expect("write lock.yaml");
}
Lockfile::unsupported_version_at(&path)
.expect("read lock.yaml")
.map(|version| version.to_string())
};
assert_eq!(version_at(None), None);
assert_eq!(version_at(Some("lockfileVersion: '6.0'\npackages: {}\n")), Some("6.0".to_string()));
assert_eq!(version_at(Some(MAIN_DOC)), None);
assert_eq!(version_at(Some("packages: [")), None);
}
+18 -1
View File
@@ -39,6 +39,9 @@ pub enum PruneError {
#[display("Failed to read CAS loader references: {_0}")]
#[diagnostic(code(ERR_PNPM_STORE_LOADER_REFERENCES))]
LoaderReferences(#[error(source)] io::Error),
#[display("Failed to read the store files that registered projects use: {_0}")]
#[diagnostic(code(ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES))]
InUseFiles(#[error(source)] io::Error),
#[diagnostic(transparent)]
StoreLock(#[error(source)] StoreLockError),
@@ -109,9 +112,23 @@ impl StoreDir {
///
/// [#344]: https://github.com/pnpm/pacquet/issues/344
pub fn prune(&self, cache_dir: &Path, protected_dirs: &[&Path]) -> Result<(), PruneError> {
self.prune_keeping(cache_dir, protected_dirs, || Ok(HashSet::new()))
}
/// [`Self::prune`], keeping every content-addressable file that
/// `in_use_files` returns whatever its hard-link count. `in_use_files`
/// runs while the prune holds the store lock, and its error stops the
/// prune before it removes anything.
pub fn prune_keeping(
&self,
cache_dir: &Path,
protected_dirs: &[&Path],
in_use_files: impl FnOnce() -> Result<HashSet<PathBuf>, PruneError>,
) -> Result<(), PruneError> {
let _store_lock = self.lock_for_prune().map_err(PruneError::StoreLock)?;
let references = crate::loader_references::loader_references(self)
let mut references = crate::loader_references::loader_references(self)
.map_err(PruneError::LoaderReferences)?;
references.files.extend(in_use_files()?);
self.prune_global_virtual_store(&references.package_roots)?;
self.remove_cached_metadata(cache_dir, protected_dirs)?;
let stats = crate::prune_cas::prune_cas_with_references(self, &references.files)
+37
View File
@@ -265,3 +265,40 @@ fn loaded_project_with_nonhidden_local_store_does_not_mark_unused_store_slots()
assert!(!dead_child.exists());
}
}
#[test]
fn prune_keeping_keeps_in_use_files_and_their_rows() {
let root = tempdir().unwrap();
let store = StoreDir::new(root.path().join("store"));
let digest = |head: &str| format!("{head}{}", "a".repeat(126));
let in_use = store.file_path_by_hex_str(&digest("01"), "");
let unused = store.file_path_by_hex_str(&digest("02"), "");
let index = crate::StoreIndex::open_in(&store).unwrap();
for (key, path, digest) in
[("in-use", &in_use, digest("01")), ("unused", &unused, digest("02"))]
{
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, "{}").unwrap();
let file = crate::CafsFileInfo { digest, mode: 0o644, size: 2, checked_at: None };
let row = crate::PackageFilesIndex {
files: std::iter::once(("package.json".to_string(), file)).collect(),
..crate::PackageFilesIndex::default()
};
index.set(key, &row).unwrap();
}
drop(index);
store
.prune_keeping(&store.root().join("cache"), &[], || {
Ok(std::iter::once(in_use.clone()).collect())
})
.unwrap();
assert!(in_use.exists());
assert!(!unused.exists());
let keys = crate::StoreIndex::open_in(&store)
.unwrap()
.keys()
.unwrap();
assert_eq!(keys, ["in-use"]);
}
@@ -7,15 +7,17 @@ import { prepareEmpty, preparePackages } from '@pnpm/prepare'
import type { ProjectManifest, ProjectRootDir } from '@pnpm/types'
import { writeYamlFileSync } from 'write-yaml-file'
import { testDefaults } from '../utils/index.js'
const gitUtils = await import('@pnpm/network.git-utils')
jest.unstable_mockModule('@pnpm/network.git-utils', () => ({
...gitUtils,
getCurrentBranch: jest.fn(),
getBranchesContainingHead: jest.fn(() => Promise.resolve([])),
}))
const { getCurrentBranch, getBranchesContainingHead } = await import('@pnpm/network.git-utils')
const { install, mutateModules } = await import('@pnpm/installing.deps-installer')
// The test store loads `@pnpm/store.controller`, which reads lockfiles through `@pnpm/network.git-utils`.
const { testDefaults } = await import('../utils/index.js')
test('install with git-branch-lockfile = true', async () => {
prepareEmpty()
@@ -2,6 +2,7 @@ import { PnpmError } from '@pnpm/error'
export class LockfileBreakingChangeError extends PnpmError {
public filename: string
public lockfileVersion?: string
constructor (
filename: string,
opts?: {
@@ -11,6 +12,7 @@ export class LockfileBreakingChangeError extends PnpmError {
) {
super('LOCKFILE_BREAKING_CHANGE', formatMessage(filename, opts))
this.filename = filename
this.lockfileVersion = opts?.lockfileVersion
}
}
+237 -1
View File
@@ -6,8 +6,10 @@ import { assertStore } from '@pnpm/assert-store'
import { STORE_VERSION } from '@pnpm/constants'
import { dlx } from '@pnpm/exec.commands'
import { prepare, prepareEmpty } from '@pnpm/prepare'
import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
import { store } from '@pnpm/store.commands'
import { REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock'
import { StoreIndex } from '@pnpm/store.index'
import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock'
import { rimrafSync } from '@zkochan/rimraf'
import { safeExeca as execa } from 'execa'
@@ -787,3 +789,237 @@ test('prune keeps the lockfile verification log', async () => {
expect(fs.existsSync(verificationLog)).toBeTruthy()
})
describe('prune keeps the packages that a registered project still uses', () => {
// The patch adds a file, so pnpm caches it as a side-effects entry of the row.
const MARKER_PATCH = [
'diff --git a/patched-marker.txt b/patched-marker.txt',
'new file mode 100644',
'index 0000000..3f2e1d4',
'--- /dev/null',
'+++ b/patched-marker.txt',
'@@ -0,0 +1 @@',
'+patched',
'',
].join('\n')
function prepareProject (workspaceYaml: string): void {
prepare({
dependencies: {
'@pnpm.e2e/pkg-with-1-dep': '100.0.0',
'is-positive': '1.0.0',
},
})
fs.writeFileSync('pnpm-workspace.yaml', workspaceYaml)
}
// The store is outside the project, so the install registers the project.
async function install (): Promise<string> {
const storeDir = path.resolve('..', 'store')
await execa('node', [pnpmBin, 'install', `--store-dir=${storeDir}`, `--cache-dir=${path.resolve('..', 'cache')}`, `--registry=${REGISTRY}`])
return path.join(storeDir, STORE_VERSION)
}
async function prune (storeDir: string, reporter?: (logObj: unknown) => void): Promise<void> {
await store.handler({
cacheDir: path.resolve('..', 'cache'),
dir: process.cwd(),
pnpmHomeDir: '',
configByUri: {},
registriesByScope: { default: REGISTRY },
reporter,
storeDir,
dlxCacheMaxAge: Infinity,
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
}, ['prune'])
}
function withStoreIndex<Result> (storeDir: string, read: (storeIndex: StoreIndex) => Result): Result {
const storeIndex = new StoreIndex(storeDir)
try {
return read(storeIndex)
} finally {
storeIndex.close()
}
}
function sortedIndexKeys (storeDir: string): string[] {
return withStoreIndex(storeDir, (storeIndex) => Array.from(storeIndex.keys()).sort())
}
// Every store file the rows under `keys` name, with their side effects.
function blobPathsOfRows (storeDir: string, keys: string[]): string[] {
return withStoreIndex(storeDir, (storeIndex) => keys.flatMap((key) => {
const row = storeIndex.get(key) as PackageFilesIndex
return [
...row.files.values(),
...Array.from(row.sideEffects?.values() ?? []).flatMap((diff) => Array.from(diff.added?.values() ?? [])),
].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode))
}))
}
async function expectPruneKeepsEveryRowAndBlob (storeDir: string): Promise<void> {
const keysBefore = sortedIndexKeys(storeDir)
expect(keysBefore.length).toBeGreaterThan(1)
const blobPaths = blobPathsOfRows(storeDir, keysBefore)
await prune(storeDir)
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
}
test('a patched dependency installed with packageImportMethod: hardlink', async () => {
// A patched package gets its own copy of the files even under `hardlink`,
// so no store file of it has a hard link outside the store.
prepareProject('packageImportMethod: hardlink\npatchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n')
fs.mkdirSync('patches')
fs.writeFileSync('patches/is-positive@1.0.0.patch', MARKER_PATCH)
const storeDir = await install()
const row = withStoreIndex(storeDir, (storeIndex) => {
const key = Array.from(storeIndex.keys()).find((key) => key.endsWith('\tis-positive@1.0.0'))!
return storeIndex.get(key) as PackageFilesIndex
})
const sideEffects = Array.from(row.sideEffects?.entries() ?? [])
expect(sideEffects.filter(([key]) => key.includes(';patch='))).toHaveLength(1)
const blobPaths = [
...row.files.values(),
...sideEffects.flatMap(([, diff]) => Array.from(diff.added?.values() ?? [])),
].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode))
const keysBefore = sortedIndexKeys(storeDir)
await prune(storeDir)
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
})
test('every package installed with packageImportMethod: copy', async () => {
// No store file has a hard link outside the store, so the hard-link
// count alone marks every package unused.
prepareProject('packageImportMethod: copy\n')
const storeDir = await install()
await expectPruneKeepsEveryRowAndBlob(storeDir)
})
// Without a wanted lockfile, the install writes the installed lockfile into
// the custom virtual store directory. Under a global virtual store, the
// project records the shared store as its virtualStoreDir and keeps the
// installed lockfile in node_modules/.pnpm.
test.each([
['modulesDir: pnpm_modules\n', 'pnpm_modules/.pnpm/lock.yaml'],
['virtualStoreDir: vs\n', 'node_modules/.pnpm/lock.yaml'],
['virtualStoreDir: vs\nlockfile: false\n', 'vs/lock.yaml'],
['enableGlobalVirtualStore: true\n', 'node_modules/.pnpm/lock.yaml'],
])('every package installed with packageImportMethod: copy and %j', async (workspaceYaml, installedLockfile) => {
prepareProject(`packageImportMethod: copy\n${workspaceYaml}`)
const storeDir = await install()
expect(fs.existsSync(installedLockfile)).toBe(true)
await expectPruneKeepsEveryRowAndBlob(storeDir)
})
test('every package installed with packageImportMethod: copy and an absolute modulesDir', async () => {
prepareProject('packageImportMethod: copy\n')
const modulesDir = path.resolve('..', 'absolute_modules')
fs.appendFileSync('pnpm-workspace.yaml', `modulesDir: ${JSON.stringify(modulesDir)}\n`)
const storeDir = await install()
expect(fs.existsSync(path.join(modulesDir, '.pnpm', 'lock.yaml'))).toBe(true)
await expectPruneKeepsEveryRowAndBlob(storeDir)
})
// Each project of a workspace without a shared lockfile is registered on
// its own, and the workspace-level modulesDir applies to it.
test('every package of a workspace project with its own lockfile and a workspace-level modulesDir', async () => {
prepare({ name: 'root' })
fs.writeFileSync('pnpm-workspace.yaml', 'packages: [project]\nsharedWorkspaceLockfile: false\nmodulesDir: pnpm_modules\npackageImportMethod: copy\n')
fs.mkdirSync('project')
fs.writeFileSync('project/package.json', JSON.stringify({
name: 'project',
dependencies: { '@pnpm.e2e/pkg-with-1-dep': '100.0.0', 'is-positive': '1.0.0' },
}))
const storeDir = await install()
expect(fs.existsSync('project/pnpm_modules/.pnpm/lock.yaml')).toBe(true)
await expectPruneKeepsEveryRowAndBlob(storeDir)
})
test('a runtime installed with packageImportMethod: copy', async () => {
// A registry tarball stands in for the runtime archive, so the install
// needs no network. The variant targets this host.
prepare({ dependencies: { node: 'runtime:22.0.0' } })
fs.writeFileSync('pnpm-workspace.yaml', 'packageImportMethod: copy\n')
const target = { os: process.platform, cpu: process.arch }
fs.writeFileSync('pnpm-lock.yaml', JSON.stringify({
lockfileVersion: '9.0',
settings: { autoInstallPeers: true, excludeLinksFromLockfile: false },
importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } },
packages: {
'node@runtime:22.0.0': {
hasBin: true,
resolution: {
type: 'variations',
variants: [{
targets: [target],
resolution: {
type: 'binary',
archive: 'tarball',
url: `${REGISTRY}is-positive/-/is-positive-1.0.0.tgz`,
integrity: getIntegrity('is-positive', '1.0.0'),
bin: { node: 'index.js' },
},
}],
},
version: '22.0.0',
},
},
snapshots: { 'node@runtime:22.0.0': {} },
}))
const storeDir = await install()
const keysBefore = sortedIndexKeys(storeDir)
const blobPaths = withStoreIndex(storeDir, (storeIndex) => keysBefore
.filter((key) => key.endsWith('\tnode@runtime:22.0.0'))
.flatMap((key) => Array.from((storeIndex.get(key) as PackageFilesIndex).files.values()))
.map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode)))
expect(blobPaths.length).toBeGreaterThan(0)
await prune(storeDir)
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
})
test('a project installed with an unsupported lockfile version is skipped with a warning', async () => {
prepareProject('packageImportMethod: copy\n')
const storeDir = await install()
// A lockfile written by a pnpm that uses another lockfile format, such as pnpm 8.
fs.writeFileSync('node_modules/.pnpm/lock.yaml', [
"lockfileVersion: '6.0'",
'',
'dependencies:',
' is-positive:',
' specifier: 1.0.0',
' version: 1.0.0',
'',
'packages:',
'',
' /is-positive@1.0.0:',
` resolution: {integrity: ${getIntegrity('is-positive', '1.0.0')}}`,
' dev: false',
'',
].join('\n'))
const reporter = jest.fn()
await prune(storeDir, reporter)
const warnings = reporter.mock.calls
.map(([logObj]) => logObj as { level?: string, message?: string })
.filter(({ level }) => level === 'warn')
.map(({ message }) => message)
expect(warnings).toStrictEqual([
`Skipped ${process.cwd()} because it was installed with lockfileVersion 6.0, which this version of pnpm does not support`,
])
})
})
+7
View File
@@ -44,18 +44,25 @@
".test": "pn pretest && cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest"
},
"dependencies": {
"@pnpm/constants": "workspace:*",
"@pnpm/crypto.hash": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fetching.fetcher-base": "workspace:*",
"@pnpm/hooks.types": "workspace:*",
"@pnpm/installing.modules-yaml": "workspace:*",
"@pnpm/installing.package-requester": "workspace:*",
"@pnpm/lockfile.fs": "workspace:*",
"@pnpm/lockfile.types": "workspace:*",
"@pnpm/lockfile.utils": "workspace:*",
"@pnpm/resolving.resolver-base": "workspace:*",
"@pnpm/store.cafs": "workspace:*",
"@pnpm/store.controller-types": "workspace:*",
"@pnpm/store.create-cafs-store": "workspace:*",
"@pnpm/store.index": "workspace:*",
"@pnpm/types": "workspace:*",
"@pnpm/workspace.workspace-manifest-reader": "workspace:*",
"@zkochan/rimraf": "catalog:",
"empathic": "catalog:",
"is-subdir": "catalog:",
"pretty-bytes": "catalog:",
"ramda": "catalog:",
@@ -0,0 +1,215 @@
import path from 'node:path'
import { LOCKFILE_VERSION, WORKSPACE_MANIFEST_FILENAME } from '@pnpm/constants'
import { isError, PnpmError } from '@pnpm/error'
import { type Modules, readModulesManifest } from '@pnpm/installing.modules-yaml'
import { readCurrentLockfile } from '@pnpm/lockfile.fs'
import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
import { packageIdFromSnapshot } from '@pnpm/lockfile.utils'
import { globalWarn } from '@pnpm/logger'
import { getFilePathByModeInCafs, type PackageFileInfo, type PackageFilesIndex } from '@pnpm/store.cafs'
import { gitHostedStoreIndexKey, pickStoreIndexKey, type StoreIndex, storeIndexKey } from '@pnpm/store.index'
import type { DepPath } from '@pnpm/types'
import { readWorkspaceManifestKeys } from '@pnpm/workspace.workspace-manifest-reader'
import * as find from 'empathic/find'
import { getRegisteredProjects } from './projectRegistry.js'
/**
* Every file of the store index rows of the packages in each registered
* project's installed lockfile, including the files of every side-effects
* entry. A package imported by copy or clone, such as a patched or built
* one, has no hard link to the store, so the prune needs this list to keep
* it. A project without an installed lockfile, a project whose lockfile
* version this pnpm does not support, and a package without a row add
* nothing.
*/
export async function getInUseFiles (storeDir: string, storeIndex: StoreIndex): Promise<Set<string>> {
const { keys, binaryKeys } = await registeredProjectsIndexKeys(storeDir)
for (const key of manifestIndexKeys(storeIndex, storeDir, binaryKeys)) keys.add(key)
const files = new Set<string>()
for (const key of keys) {
const row = readIndexRow(storeIndex, storeDir, key)
if (row == null) continue
for (const { digest, mode } of rowFiles(row)) {
files.add(path.normalize(getFilePathByModeInCafs(storeDir, digest, mode)))
}
}
return files
}
interface IndexKey {
key: string
binary: boolean
}
async function registeredProjectsIndexKeys (storeDir: string): Promise<{ keys: Set<string>, binaryKeys: Set<string> }> {
const keys = new Set<string>()
const binaryKeys = new Set<string>()
for (const projectDir of await getRegisteredProjects(storeDir)) {
// eslint-disable-next-line no-await-in-loop -- hold one project's parsed lockfiles at a time
for (const { key, binary } of await projectIndexKeys(projectDir)) {
keys.add(key)
if (binary) binaryKeys.add(key)
}
}
return { keys, binaryKeys }
}
async function projectIndexKeys (projectDir: string): Promise<IndexKey[]> {
return (await readInstalledLockfiles(projectDir)).flatMap(lockfileIndexKeys)
}
function lockfileIndexKeys (lockfile: LockfileObject): IndexKey[] {
return Object.entries(lockfile.packages ?? {}).flatMap(([depPath, snapshot]) =>
packageIndexKeys(snapshot.resolution, packageIdFromSnapshot(depPath as DepPath, snapshot))
)
}
/**
* pnpm 12 shares this store and keys the row of a runtime as
* `package-manifest\t<manifest integrity>\t<binary key>`. The lockfile has
* no manifest integrity, so this scans the keys, but only when the
* lockfiles name a binary.
*/
function manifestIndexKeys (storeIndex: StoreIndex, storeDir: string, binaryKeys: Set<string>): string[] {
if (binaryKeys.size === 0) return []
const prefix = 'package-manifest\t'
const keys: string[] = []
try {
for (const key of storeIndex.keys()) {
if (!key.startsWith(prefix)) continue
const separator = key.indexOf('\t', prefix.length)
if (separator !== -1 && binaryKeys.has(key.slice(separator + 1))) keys.push(key)
}
} catch (err: unknown) {
throw inUseFilesError(storeDir, err)
}
return keys
}
function readIndexRow (storeIndex: StoreIndex, storeDir: string, key: string): PackageFilesIndex | undefined {
try {
return storeIndex.get(key) as PackageFilesIndex | undefined
} catch (err: unknown) {
throw inUseFilesError(storeDir, err)
}
}
/**
* The lockfiles the last install wrote, which record what is installed in
* the project.
*
* The install writes the lockfile to `<modulesDir>/.pnpm`, or to the
* `virtualStoreDir` that `<modulesDir>/.modules.yaml` records when the
* project has no wanted lockfile. Under a global virtual store, that
* setting names the shared store. The manifest does not say which case
* applies, so the prune reads both.
*/
async function readInstalledLockfiles (projectDir: string): Promise<LockfileObject[]> {
const modulesDir = path.resolve(projectDir, await readConfiguredModulesDir(projectDir))
const defaultDir = path.join(modulesDir, '.pnpm')
const modules = await readModulesManifestOfProject(modulesDir)
const installStateDirs = new Set([defaultDir, modules?.virtualStoreDir ?? defaultDir])
const lockfiles = await Promise.all(Array.from(installStateDirs, async (installStateDir) => readInstalledLockfile(projectDir, installStateDir)))
return lockfiles.filter((lockfile) => lockfile != null)
}
/**
* The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the
* project sets, or `node_modules`. A `modulesDir` set only on the command
* line, in the environment, in the global config, or in `packageConfigs`
* is not seen.
*/
async function readConfiguredModulesDir (projectDir: string): Promise<string> {
try {
// `findWorkspaceDir` would also honor the workspace directory that the
// environment of the prune sets, which names no registered project's
// workspace.
const workspaceManifest = find.file(WORKSPACE_MANIFEST_FILENAME, { cwd: projectDir })
if (workspaceManifest == null) return 'node_modules'
const workspaceDir = path.dirname(workspaceManifest)
// `PnpmSettings` does not declare `modulesDir`.
const manifest = await readWorkspaceManifestKeys(workspaceDir, MODULES_DIR_SETTING) as Record<string, unknown> | undefined
const modulesDir = manifest?.modulesDir
return typeof modulesDir === 'string' ? modulesDir : 'node_modules'
} catch (err: unknown) {
throw inUseFilesError(projectDir, err)
}
}
const MODULES_DIR_SETTING: ReadonlySet<string> = new Set(['modulesDir'])
async function readModulesManifestOfProject (modulesDir: string): Promise<Modules | null> {
try {
return await readModulesManifest(modulesDir)
} catch (err: unknown) {
throw inUseFilesError(modulesDir, err)
}
}
/**
* A lockfile of a version this pnpm cannot read adds nothing, like a
* missing one, so one project installed by another pnpm does not stop the
* prune. The hard-link check then decides for that project's files.
*/
async function readInstalledLockfile (projectDir: string, installStateDir: string): Promise<LockfileObject | null> {
try {
return await readCurrentLockfile(installStateDir, { ignoreIncompatible: false, wantedVersions: [LOCKFILE_VERSION] })
} catch (err: unknown) {
const lockfileVersion = unsupportedLockfileVersion(err)
if (lockfileVersion != null) {
globalWarn(`Skipped ${projectDir} because it was installed with lockfileVersion ${lockfileVersion}, which this version of pnpm does not support`)
return null
}
throw inUseFilesError(installStateDir, err)
}
}
function unsupportedLockfileVersion (err: unknown): string | undefined {
if (!isError(err) || !('code' in err) || err.code !== 'ERR_PNPM_LOCKFILE_BREAKING_CHANGE') return undefined
return (err as { lockfileVersion?: string }).lockfileVersion
}
function inUseFilesError (location: string, err: unknown): PnpmError {
return new PnpmError('STORE_DIR_PRUNE_IN_USE_FILES',
`Failed to read the store files that registered projects use: ${location}: ${isError(err) ? err.message : String(err)}`)
}
/**
* The lockfile does not say which key the install used for a git-hosted
* package, which has a key for its contents with and without a build, or for
* a variations resolution, whose variant depends on the installing host
* and its `supportedArchitectures`.
*/
function packageIndexKeys (resolution: LockfileResolution, pkgId: string): IndexKey[] {
if (!('type' in resolution) || resolution.type == null) {
if ('tarball' in resolution) {
return [true, false].map((built) => packageKey(pickStoreIndexKey(resolution, pkgId, { built })))
}
return [packageKey(storeIndexKey(resolution.integrity, pkgId))]
}
switch (resolution.type) {
case 'git':
return [true, false].map((built) => packageKey(gitHostedStoreIndexKey(pkgId, { built })))
case 'binary':
return [{ key: storeIndexKey(resolution.integrity, pkgId), binary: true }]
case 'variations':
return resolution.variants.flatMap((variant) => packageIndexKeys(variant.resolution, pkgId))
default:
// A directory has no row, and the content of a custom one is not
// known here.
return []
}
}
function packageKey (key: string): IndexKey {
return { key, binary: false }
}
function * rowFiles (row: PackageFilesIndex): Iterable<PackageFileInfo> {
yield * row.files.values()
for (const sideEffectsDiff of row.sideEffects?.values() ?? []) {
if (sideEffectsDiff.added != null) yield * sideEffectsDiff.added.values()
}
}
@@ -8,6 +8,7 @@ import type { StoreIndex } from '@pnpm/store.index'
import { rimraf } from '@zkochan/rimraf'
import prettyBytes from 'pretty-bytes'
import { getInUseFiles } from './inUseFiles.js'
import { pruneGlobalVirtualStore } from './pruneGlobalVirtualStore.js'
const BIG_ONE = BigInt(1) as unknown
@@ -19,6 +20,10 @@ export interface PruneOptions {
}
export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, removeAlienFiles?: boolean): Promise<void> {
// 0. Collect the files that registered projects still use, before any
// removal, so an unreadable lockfile stops the prune untouched
const inUseFiles = await getInUseFiles(storeDir, storeIndex)
// 1. First, prune the global virtual store
// This must happen BEFORE pruning the CAS, because removing packages from
// the virtual store will reduce hard link counts on files in the CAS
@@ -29,7 +34,7 @@ export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, r
await removeStaleSpillDirectories(storeDir)
// 3. Prune the content-addressable store (CAS)
const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles)
const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles, inUseFiles)
// 4. Clean up orphaned package index entries
removeOrphanedIndexEntries(storeIndex, removedHashes)
@@ -89,14 +94,16 @@ interface CafsPruneStats {
fileCounter: number
totalSize: number
removedHashes: Set<string>
inUseFiles: Set<string>
}
async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined): Promise<Set<string>> {
async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined, inUseFiles: Set<string>): Promise<Set<string>> {
const cafsDir = path.join(storeDir, 'files')
const stats: CafsPruneStats = {
fileCounter: 0,
totalSize: 0,
removedHashes: new Set<string>(),
inUseFiles,
}
const dirs = await getSubdirsSafely(cafsDir)
await Promise.all(dirs.map(async (dir) => {
@@ -130,7 +137,7 @@ async function pruneCafsEntry ({ dir, fileName, subdir, removeAlienFiles, stats
}
return
}
if (stat.nlink === 1 || stat.nlink === BIG_ONE) {
if ((stat.nlink === 1 || stat.nlink === BIG_ONE) && !stats.inUseFiles.has(filePath)) {
stats.totalSize += stat.size
await fs.unlink(filePath)
stats.fileCounter++
+90 -1
View File
@@ -1,10 +1,12 @@
import { promises as fs } from 'node:fs'
import { existsSync, promises as fs } from 'node:fs'
import path from 'node:path'
import { afterEach, expect, jest, test } from '@jest/globals'
import { getFilePathByModeInCafs } from '@pnpm/store.cafs'
import { StoreIndex } from '@pnpm/store.index'
import { temporaryDirectory } from 'tempy'
import { registerProject } from '../src/storeController/projectRegistry.js'
import { prune } from '../src/storeController/prune.js'
const temporaryDirectories: string[] = []
@@ -124,6 +126,93 @@ test('prune propagates unexpected spill directory errors', async () => {
await expect(prune(options)).rejects.toBe(error)
})
test.each([
['installed lockfile', path.join('node_modules', '.pnpm', 'lock.yaml'), 'packages: ['],
['modules manifest', path.join('node_modules', '.modules.yaml'), '{'],
])('prune fails before it removes anything when a registered project has a broken %s', async (_name, brokenFile, content) => {
const options = createPruneOptions()
const projectDir = path.join(options.cacheDir, '..', 'project')
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
await fs.writeFile(path.join(projectDir, brokenFile), content)
await registerProject(options.storeDir, projectDir)
const unusedFile = path.join(options.storeDir, 'files', '00', 'unused')
await fs.mkdir(path.dirname(unusedFile), { recursive: true })
await fs.writeFile(unusedFile, 'unused')
await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' })
expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused')
})
test('prune fails before it removes anything when the store index cannot be read', async () => {
const options = createPruneOptions()
const projectDir = path.join(options.cacheDir, '..', 'project')
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), [
"lockfileVersion: '9.0'",
'importers:',
" '.': {}",
'packages:',
' is-positive@1.0.0:',
' resolution: {integrity: sha512-AAAA}',
'snapshots:',
' is-positive@1.0.0: {}',
'',
].join('\n'))
await registerProject(options.storeDir, projectDir)
const unusedFile = path.join(options.storeDir, 'files', '00', 'unused')
await fs.mkdir(path.dirname(unusedFile), { recursive: true })
await fs.writeFile(unusedFile, 'unused')
jest.spyOn(options.storeIndex, 'get').mockImplementation(() => {
throw new Error('database disk image is malformed')
})
await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' })
expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused')
})
test('prune keeps a runtime whose index row is keyed with its manifest integrity', async () => {
// pnpm 12 shares this store and keys a runtime row as
// `package-manifest\t<manifest integrity>\t<integrity>\t<pkgId>`.
const options = createPruneOptions()
const projectDir = path.join(options.cacheDir, '..', 'project')
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
const integrity = 'sha512-BBBB'
await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), JSON.stringify({
lockfileVersion: '9.0',
importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } },
packages: {
'node@runtime:22.0.0': {
resolution: {
type: 'variations',
variants: [{
targets: [{ os: 'linux', cpu: 'x64' }],
resolution: { type: 'binary', archive: 'tarball', url: 'https://example.com/node.tgz', integrity, bin: { node: 'bin/node' } },
}],
},
version: '22.0.0',
},
},
snapshots: { 'node@runtime:22.0.0': {} },
}))
await registerProject(options.storeDir, projectDir)
const files = new Map([
['package.json', { digest: 'aa'.repeat(32), mode: 0o644, size: 2 }],
['bin/node', { digest: 'bb'.repeat(32), mode: 0o755, size: 2 }],
])
const blobPaths = Array.from(files.values(), ({ digest, mode }) => getFilePathByModeInCafs(options.storeDir, digest, mode))
await Promise.all(blobPaths.map(async (blobPath) => {
await fs.mkdir(path.dirname(blobPath), { recursive: true })
await fs.writeFile(blobPath, '{}')
}))
const key = `package-manifest\tsha512-CCCC\t${integrity}\tnode@runtime:22.0.0`
options.storeIndex.set(key, { algo: 'sha512', files })
await prune(options)
expect(options.storeIndex.has(key)).toBe(true)
expect(blobPaths.filter((blobPath) => !existsSync(blobPath))).toStrictEqual([])
})
function createPruneOptions () {
const directory = temporaryDirectory()
temporaryDirectories.push(directory)
+18
View File
@@ -12,6 +12,9 @@
{
"path": "../../__utils__/prepare"
},
{
"path": "../../core/constants"
},
{
"path": "../../core/error"
},
@@ -33,15 +36,30 @@
{
"path": "../../installing/client"
},
{
"path": "../../installing/modules-yaml"
},
{
"path": "../../installing/package-requester"
},
{
"path": "../../lockfile/fs"
},
{
"path": "../../lockfile/types"
},
{
"path": "../../lockfile/utils"
},
{
"path": "../../resolving/resolver-base"
},
{
"path": "../../worker"
},
{
"path": "../../workspace/workspace-manifest-reader"
},
{
"path": "../cafs"
},