fix(store): keep packages that a project still uses on store prune (#17327)
The CAS prune removed a store file whose hard-link count was 1. A package imported with clone or copy leaves its store files at one link while a project still uses it. That is every patched or built package, every package under packageImportMethod: copy, and every package of a macOS clone. So the prune removed them, and the next install downloaded them again. The prune now also keeps every file that a registered project's installed lockfile references: - For each registered project it reads <modulesDir>/.pnpm/lock.yaml and the lock.yaml in the virtualStoreDir recorded in <modulesDir>/.modules.yaml. modulesDir comes from the nearest pnpm-workspace.yaml at or above the registered directory. - Each package's store index keys come from the install's own key functions: both keys of a git-hosted package, the manifest-keyed and plain keys of a runtime, and every variant of a variations resolution. - Every file of each row is kept, including each side-effects entry. - A project with no installed lockfile and a package with no row add nothing, so the hard-link check still decides for them. - A lockfile of an unsupported version is skipped with a warning. A lockfile, .modules.yaml, pnpm-workspace.yaml, or store index row that cannot be read fails the prune with ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES. Registry errors keep their own codes. In pnpm 12, StoreDir::prune_keeping runs a caller's collector under the prune lock, and the CLI supplies the collector because store-dir does not depend on the lockfile crate. pnpm 11 collects before any removal. pnpm 11 and pnpm 12 share one store. pnpm 12 keys a runtime row with its synthesized manifest, so the pnpm 11 collector also keeps package-manifest keys whose tail it references. It scans the index keys only when a lockfile holds a runtime. A modulesDir set only on the command line, in the environment, in the global config, or in packageConfigs is not seen. --------- Co-authored-by: Zoltan Kochan <z@kochan.io> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
21 files changed
+1346
-16
No files matched your search
@@ -0,0 +1,8 @@
|
||||
---
|
||||
"@pnpm/lockfile.fs": patch
|
||||
"@pnpm/store.controller": patch
|
||||
"pnpm": patch
|
||||
"pacquet": patch
|
||||
---
|
||||
|
||||
`pnpm store prune` now keeps the packages that a project still uses when pnpm copied or cloned them into `node_modules`. These include patched packages, built packages, and every package installed with `packageImportMethod: copy`. Before, the prune removed them, and the next install downloaded them again [#7192](https://github.com/pnpm/pnpm/issues/7192).
|
||||
Generated
+21
@@ -10173,6 +10173,9 @@ importers:
|
||||
|
||||
pnpm11/store/controller:
|
||||
dependencies:
|
||||
'@pnpm/constants':
|
||||
specifier: workspace:*
|
||||
version: link:../../core/constants
|
||||
'@pnpm/crypto.hash':
|
||||
specifier: workspace:*
|
||||
version: link:../../crypto/hash
|
||||
@@ -10185,9 +10188,21 @@ importers:
|
||||
'@pnpm/hooks.types':
|
||||
specifier: workspace:*
|
||||
version: link:../../hooks/types
|
||||
'@pnpm/installing.modules-yaml':
|
||||
specifier: workspace:*
|
||||
version: link:../../installing/modules-yaml
|
||||
'@pnpm/installing.package-requester':
|
||||
specifier: workspace:*
|
||||
version: link:../../installing/package-requester
|
||||
'@pnpm/lockfile.fs':
|
||||
specifier: workspace:*
|
||||
version: link:../../lockfile/fs
|
||||
'@pnpm/lockfile.types':
|
||||
specifier: workspace:*
|
||||
version: link:../../lockfile/types
|
||||
'@pnpm/lockfile.utils':
|
||||
specifier: workspace:*
|
||||
version: link:../../lockfile/utils
|
||||
'@pnpm/resolving.resolver-base':
|
||||
specifier: workspace:*
|
||||
version: link:../../resolving/resolver-base
|
||||
@@ -10209,9 +10224,15 @@ importers:
|
||||
'@pnpm/worker':
|
||||
specifier: workspace:^
|
||||
version: link:../../worker
|
||||
'@pnpm/workspace.workspace-manifest-reader':
|
||||
specifier: workspace:*
|
||||
version: link:../../workspace/workspace-manifest-reader
|
||||
'@zkochan/rimraf':
|
||||
specifier: 'catalog:'
|
||||
version: 4.0.0
|
||||
empathic:
|
||||
specifier: 'catalog:'
|
||||
version: 2.1.0
|
||||
is-subdir:
|
||||
specifier: 'catalog:'
|
||||
version: 2.0.1
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! any project.
|
||||
|
||||
mod add;
|
||||
mod in_use_files;
|
||||
mod status;
|
||||
|
||||
use crate::{
|
||||
@@ -80,7 +81,11 @@ fn prune(config: &Config) -> miette::Result<()> {
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
config.store_dir.prune(&config.cache_dir, &protected_dirs).wrap_err("pruning store")?;
|
||||
config.store_dir
|
||||
.prune_keeping(&config.cache_dir, &protected_dirs, || {
|
||||
in_use_files::in_use_files(&config.store_dir)
|
||||
})
|
||||
.wrap_err("pruning store")?;
|
||||
remove_orphaned_private_engine_installs(config)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! The store files of the packages that registered projects installed.
|
||||
//!
|
||||
//! A package imported by copy or clone, such as a patched or built one, has
|
||||
//! no hard link to the store, so `store prune` needs this list to keep it.
|
||||
|
||||
use pnpm_config::workspace_root_or;
|
||||
use pnpm_deps_restorer::{binary_store_index_key, store_index_key_for_resolution};
|
||||
use pnpm_fs::lexical_normalize;
|
||||
use pnpm_lockfile::{Lockfile, LockfileResolution, PackageKey};
|
||||
use pnpm_modules_yaml::{Host, read_modules_layout};
|
||||
use pnpm_store_dir::{
|
||||
CafsFileInfo, PackageFilesIndex, PruneError, StoreDir, StoreIndex, StoreIndexError,
|
||||
decode_package_files_index, get_registered_projects,
|
||||
};
|
||||
use pnpm_workspace::read_workspace_manifest_document;
|
||||
use serde::Deserialize;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
io, iter,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// Every file of the store index rows of the packages in each registered
|
||||
/// project's current lockfile, including the files of every side-effects
|
||||
/// entry. A project without a current lockfile, a project whose lockfile
|
||||
/// version this pnpm does not support, and a package without a row add
|
||||
/// nothing. A registry error keeps its own code, and any other
|
||||
/// failure is [`PruneError::InUseFiles`].
|
||||
pub(super) fn in_use_files(store_dir: &StoreDir) -> Result<HashSet<PathBuf>, PruneError> {
|
||||
if !StoreIndex::exists(store_dir.root()) {
|
||||
return Ok(HashSet::new());
|
||||
}
|
||||
let mut keys = HashSet::new();
|
||||
for project in get_registered_projects(store_dir).map_err(PruneError::ListProjects)? {
|
||||
// The registry joins a relative link target without normalizing it.
|
||||
let project = lexical_normalize(&project);
|
||||
for lockfile in load_installed_lockfiles(&project).map_err(PruneError::InUseFiles)? {
|
||||
keys.extend(package_index_keys(&lockfile).map_err(PruneError::InUseFiles)?);
|
||||
}
|
||||
}
|
||||
if keys.is_empty() {
|
||||
return Ok(HashSet::new());
|
||||
}
|
||||
let keys: Vec<String> = keys.into_iter().collect();
|
||||
let rows = read_rows(store_dir, &keys)
|
||||
.map_err(|error| PruneError::InUseFiles(io::Error::other(error)))?;
|
||||
Ok(rows
|
||||
.iter()
|
||||
.flat_map(row_files)
|
||||
.filter_map(|file| store_dir.cas_file_path_by_mode(&file.digest, file.mode))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Fails on an undecodable row, which [`StoreIndex::get_many`] skips,
|
||||
/// because skipping it would let the prune remove the row's files.
|
||||
fn read_rows(
|
||||
store_dir: &StoreDir,
|
||||
keys: &[String],
|
||||
) -> Result<Vec<PackageFilesIndex>, StoreIndexError> {
|
||||
StoreIndex::open_readonly_in(store_dir)?
|
||||
.get_many_raw(keys)?
|
||||
.iter()
|
||||
.map(|(_, bytes)| decode_package_files_index(bytes))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The lockfiles the last install wrote, which record what is installed in
|
||||
/// the project.
|
||||
///
|
||||
/// The install writes the lockfile to `<modulesDir>/.pnpm`, or to the
|
||||
/// `virtualStoreDir` that `<modulesDir>/.modules.yaml` records when the
|
||||
/// project sets one. Under a global virtual store, that setting names the
|
||||
/// shared store and the lockfile stays in `<modulesDir>/.pnpm`. The
|
||||
/// manifest does not say which case applies, so the prune reads both.
|
||||
fn load_installed_lockfiles(project: &Path) -> io::Result<Vec<Lockfile>> {
|
||||
let modules_dir = project.join(configured_modules_dir(project)?);
|
||||
let default_dir = modules_dir.join(".pnpm");
|
||||
let virtual_store_dir = read_modules_layout::<Host>(&modules_dir)
|
||||
.map_err(io::Error::other)?
|
||||
.map(|layout| PathBuf::from(layout.virtual_store_dir))
|
||||
.filter(|dir| *dir != default_dir);
|
||||
let mut lockfiles = Vec::new();
|
||||
for install_state_dir in iter::once(default_dir).chain(virtual_store_dir) {
|
||||
lockfiles.extend(load_installed_lockfile(project, &install_state_dir)?);
|
||||
}
|
||||
Ok(lockfiles)
|
||||
}
|
||||
|
||||
/// The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the
|
||||
/// project sets, or `node_modules`. A `modulesDir` set only on the command
|
||||
/// line, in the environment, in the global config, or in `packageConfigs`
|
||||
/// is not seen.
|
||||
fn configured_modules_dir(project: &Path) -> io::Result<String> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ModulesDirSetting {
|
||||
modules_dir: Option<String>,
|
||||
}
|
||||
let Some(document) =
|
||||
read_workspace_manifest_document(&workspace_root_or(project)).map_err(io::Error::other)?
|
||||
else {
|
||||
return Ok("node_modules".to_string());
|
||||
};
|
||||
let setting: Option<ModulesDirSetting> = serde_saphyr::from_str(&document.text)
|
||||
.map_err(|error| io::Error::other(format!("{}: {error}", document.path.display())))?;
|
||||
Ok(setting
|
||||
.and_then(|setting| setting.modules_dir)
|
||||
.unwrap_or_else(|| "node_modules".to_string()))
|
||||
}
|
||||
|
||||
/// A lockfile of a version this pnpm cannot read adds nothing, like a
|
||||
/// missing one, so one project installed by another pnpm does not stop the
|
||||
/// prune. The hard-link check then decides for that project's files.
|
||||
fn load_installed_lockfile(
|
||||
project: &Path,
|
||||
install_state_dir: &Path,
|
||||
) -> io::Result<Option<Lockfile>> {
|
||||
let error = match Lockfile::load_current_from_install_state_dir(install_state_dir) {
|
||||
Ok(lockfile) => return Ok(lockfile),
|
||||
Err(error) => error,
|
||||
};
|
||||
let lockfile_path = install_state_dir.join(Lockfile::CURRENT_FILE_NAME);
|
||||
if let Ok(Some(version)) = Lockfile::unsupported_version_at(&lockfile_path) {
|
||||
pnpm_diagnostics::eprintln!(
|
||||
"Skipped {} because it was installed with lockfileVersion {version}, \
|
||||
which this version of pnpm does not support",
|
||||
project.display(),
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
Err(io::Error::other(format!("{}: {error}", install_state_dir.display())))
|
||||
}
|
||||
|
||||
fn package_index_keys(lockfile: &Lockfile) -> io::Result<Vec<String>> {
|
||||
let mut keys = Vec::new();
|
||||
for (key, metadata) in lockfile.packages.iter().flatten() {
|
||||
add_resolution_index_keys(&mut keys, key, &metadata.resolution)?;
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
/// The lockfile does not say which key the install used for:
|
||||
/// - a git-hosted package, which has a key for its built and its unbuilt
|
||||
/// contents;
|
||||
/// - a `Variations` resolution, whose variant depends on the installing
|
||||
/// host and its `supportedArchitectures`;
|
||||
/// - a runtime, whose row an older install keyed without its manifest.
|
||||
fn add_resolution_index_keys(
|
||||
keys: &mut Vec<String>,
|
||||
key: &PackageKey,
|
||||
resolution: &LockfileResolution,
|
||||
) -> io::Result<()> {
|
||||
if let LockfileResolution::Variations(variations) = resolution {
|
||||
for variant in &variations.variants {
|
||||
add_resolution_index_keys(keys, key, &variant.resolution)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let pkg_id = key.pkg_id();
|
||||
keys.extend(
|
||||
[true, false]
|
||||
.into_iter()
|
||||
.filter_map(|built| store_index_key_for_resolution(resolution, &pkg_id, built)),
|
||||
);
|
||||
if let LockfileResolution::Binary(binary) = resolution {
|
||||
keys.push(binary_store_index_key(key, binary).map_err(io::Error::other)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn row_files(row: &PackageFilesIndex) -> impl Iterator<Item = &CafsFileInfo> {
|
||||
let side_effects_files = row.side_effects
|
||||
.iter()
|
||||
.flat_map(|side_effects| side_effects.values())
|
||||
.filter_map(|diff| diff.added.as_ref())
|
||||
.flat_map(|added| added.values());
|
||||
row.files.values().chain(side_effects_files)
|
||||
}
|
||||
@@ -13,7 +13,7 @@ use std::{
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
struct RuntimeFixture {
|
||||
pub(crate) struct RuntimeFixture {
|
||||
name: &'static str,
|
||||
version: &'static str,
|
||||
archive_mock: mockito::Mock,
|
||||
@@ -963,7 +963,7 @@ fn assert_runtime_bad_integrity(name: &'static str, version: &'static str) {
|
||||
assert!(stderr.contains("Integrity check failed"), "stderr was:\n{stderr}");
|
||||
}
|
||||
|
||||
fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
|
||||
pub(crate) fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
|
||||
let workspace = root.path().join("workspace");
|
||||
fs::create_dir(&workspace).unwrap();
|
||||
fs::write(
|
||||
@@ -976,7 +976,7 @@ fn prepare_workspace(root: &TempDir, extra_yaml: &str) -> PathBuf {
|
||||
workspace
|
||||
}
|
||||
|
||||
fn runtime_fixture(
|
||||
pub(crate) fn runtime_fixture(
|
||||
server: &mut mockito::Server,
|
||||
name: &'static str,
|
||||
version: &'static str,
|
||||
@@ -1027,7 +1027,7 @@ fn runtime_fixture(
|
||||
}
|
||||
}
|
||||
|
||||
fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
|
||||
pub(crate) fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
|
||||
let dependencies = fixtures
|
||||
.iter()
|
||||
.map(|fixture| {
|
||||
@@ -1038,7 +1038,7 @@ fn write_runtime_manifest(workspace: &Path, fixtures: &[RuntimeFixture]) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) {
|
||||
pub(crate) fn write_runtime_lockfile(workspace: &Path, fixtures: &[RuntimeFixture]) {
|
||||
write_runtime_lockfile_for_group(workspace, fixtures, "dependencies");
|
||||
}
|
||||
|
||||
|
||||
@@ -192,6 +192,7 @@ mod store;
|
||||
mod store_pkg_content_check;
|
||||
mod store_placement;
|
||||
mod store_project_registry;
|
||||
mod store_prune_in_use;
|
||||
mod sync_injected_deps_after_scripts;
|
||||
mod tar_registry;
|
||||
mod tarball_url_dependency;
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
//! `pnpm store prune` must keep the store files of every package a
|
||||
//! registered project installed, including the packages that pnpm copied
|
||||
//! into `node_modules`, which hold no hard link to the store.
|
||||
|
||||
use crate::{
|
||||
_utils::enable_gvs_in_workspace_yaml,
|
||||
install_runtimes::{
|
||||
prepare_workspace, runtime_fixture, write_runtime_lockfile, write_runtime_manifest,
|
||||
},
|
||||
patch::MARKER_PATCH,
|
||||
};
|
||||
use assert_cmd::prelude::*;
|
||||
use command_extra::CommandExtra;
|
||||
use pnpm_graph_hasher::{host_arch, host_platform};
|
||||
use pnpm_store_dir::{PackageFilesIndex, StoreDir, StoreIndex};
|
||||
use pnpm_testing_utils::{
|
||||
bin::{AddMockedRegistry, CommandTempCwd},
|
||||
command_env::CommandTestExt,
|
||||
};
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
fn pacquet_at(workspace: &Path) -> Command {
|
||||
Command::cargo_bin("pnpm")
|
||||
.expect("find the pnpm binary")
|
||||
.with_current_dir(workspace)
|
||||
.without_ambient_pnpm_config()
|
||||
}
|
||||
|
||||
fn write_manifest(workspace: &Path) {
|
||||
fs::write(
|
||||
workspace.join("package.json"),
|
||||
serde_json::json!({
|
||||
"dependencies": {
|
||||
"@pnpm.e2e/pkg-with-1-dep": "100.0.0",
|
||||
"is-positive": "1.0.0",
|
||||
},
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.expect("write package.json");
|
||||
}
|
||||
|
||||
fn append_workspace_yaml(workspace: &Path, text: &str) {
|
||||
let path = workspace.join("pnpm-workspace.yaml");
|
||||
let mut yaml = fs::read_to_string(&path).expect("read pnpm-workspace.yaml");
|
||||
if !yaml.ends_with('\n') {
|
||||
yaml.push('\n');
|
||||
}
|
||||
yaml.push_str(text);
|
||||
fs::write(&path, yaml).expect("write pnpm-workspace.yaml");
|
||||
}
|
||||
|
||||
fn sorted_index_keys(store_dir: &StoreDir) -> Vec<String> {
|
||||
let mut keys = StoreIndex::open_readonly_in(store_dir)
|
||||
.expect("open the store index")
|
||||
.keys()
|
||||
.expect("read the store index keys");
|
||||
keys.sort();
|
||||
keys
|
||||
}
|
||||
|
||||
fn is_positive_row(store_dir: &StoreDir) -> PackageFilesIndex {
|
||||
let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index");
|
||||
let key = index
|
||||
.keys()
|
||||
.expect("read the store index keys")
|
||||
.into_iter()
|
||||
.find(|key| key.ends_with("\tis-positive@1.0.0"))
|
||||
.expect("a store index row for is-positive@1.0.0");
|
||||
index
|
||||
.get(&key)
|
||||
.expect("read the store index row")
|
||||
.expect("the row is present")
|
||||
}
|
||||
|
||||
/// Every store file the row names: its base files and the files each
|
||||
/// side-effects entry adds.
|
||||
fn row_blob_paths(store_dir: &StoreDir, row: &PackageFilesIndex) -> Vec<PathBuf> {
|
||||
let side_effects_files = row.side_effects
|
||||
.iter()
|
||||
.flat_map(|side_effects| side_effects.values())
|
||||
.filter_map(|diff| diff.added.as_ref())
|
||||
.flat_map(|added| added.values());
|
||||
row.files
|
||||
.values()
|
||||
.chain(side_effects_files)
|
||||
.map(|info| {
|
||||
store_dir
|
||||
.cas_file_path_by_mode(&info.digest, info.mode)
|
||||
.expect("an index row holds hex digests")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every store file the rows under `keys` name.
|
||||
fn blob_paths_of_rows(store_dir: &StoreDir, keys: &[String]) -> Vec<PathBuf> {
|
||||
let index = StoreIndex::open_readonly_in(store_dir).expect("open the store index");
|
||||
keys.iter()
|
||||
.flat_map(|key| {
|
||||
let row = index
|
||||
.get(key)
|
||||
.expect("read the store index row")
|
||||
.expect("the row is present");
|
||||
row_blob_paths(store_dir, &row)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn prune(workspace: &Path) {
|
||||
let output = pacquet_at(workspace)
|
||||
.with_args(["store", "prune"])
|
||||
.output()
|
||||
.expect("run store prune");
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
eprintln!("STDERR:\n{stderr}\n");
|
||||
assert!(output.status.success(), "store prune failed: {stderr}");
|
||||
}
|
||||
|
||||
/// A patched package gets its own copy of the files even under `hardlink`,
|
||||
/// so no store file of it has a hard link outside the store. `hardlink`
|
||||
/// keeps every other package linked on each platform, which `auto` does
|
||||
/// not do on a copy-on-write filesystem.
|
||||
#[test]
|
||||
fn store_prune_keeps_the_files_of_a_patched_dependency() {
|
||||
let CommandTempCwd {
|
||||
root: _root, workspace, npmrc_info, ..
|
||||
} = CommandTempCwd::init().add_mocked_registry();
|
||||
let AddMockedRegistry {
|
||||
mock_instance: _mock_instance,
|
||||
store_dir,
|
||||
..
|
||||
} = npmrc_info;
|
||||
write_manifest(&workspace);
|
||||
fs::create_dir_all(workspace.join("patches")).expect("create the patches dir");
|
||||
fs::write(workspace.join("patches/is-positive@1.0.0.patch"), MARKER_PATCH)
|
||||
.expect("write the patch file");
|
||||
append_workspace_yaml(
|
||||
&workspace,
|
||||
concat!(
|
||||
"packageImportMethod: hardlink\n",
|
||||
"patchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n",
|
||||
),
|
||||
);
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let store_dir = StoreDir::new(store_dir);
|
||||
let row = is_positive_row(&store_dir);
|
||||
let patched_entries = row.side_effects
|
||||
.as_ref()
|
||||
.map_or(0, |side_effects| {
|
||||
side_effects
|
||||
.keys()
|
||||
.filter(|key| key.contains(";patch="))
|
||||
.count()
|
||||
});
|
||||
assert_eq!(patched_entries, 1, "the patch must be cached as a side-effects entry");
|
||||
let blob_paths = row_blob_paths(&store_dir, &row);
|
||||
let keys_before = sorted_index_keys(&store_dir);
|
||||
|
||||
prune(&workspace);
|
||||
|
||||
assert_eq!(sorted_index_keys(&store_dir), keys_before);
|
||||
let missing: Vec<_> = blob_paths
|
||||
.iter()
|
||||
.filter(|path| !path.exists())
|
||||
.collect();
|
||||
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
|
||||
}
|
||||
|
||||
/// With `packageImportMethod: copy` no store file has a hard link outside
|
||||
/// the store, so the hard-link count alone marks every package unused.
|
||||
#[test]
|
||||
fn store_prune_keeps_every_package_installed_by_copy() {
|
||||
assert_prune_keeps_every_package_installed_by_copy(|_| {}, "node_modules/.pnpm/lock.yaml");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir() {
|
||||
assert_prune_keeps_every_package_installed_by_copy(
|
||||
|workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\n"),
|
||||
"vs/lock.yaml",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_virtual_store_dir_without_a_lockfile()
|
||||
{
|
||||
assert_prune_keeps_every_package_installed_by_copy(
|
||||
|workspace| append_workspace_yaml(workspace, "virtualStoreDir: vs\nlockfile: false\n"),
|
||||
"vs/lock.yaml",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_keeps_every_package_installed_by_copy_in_a_custom_modules_dir() {
|
||||
assert_prune_keeps_every_package_installed_by_copy(
|
||||
|workspace| append_workspace_yaml(workspace, "modulesDir: pnpm_modules\n"),
|
||||
"pnpm_modules/.pnpm/lock.yaml",
|
||||
);
|
||||
}
|
||||
|
||||
/// The project records the shared store as its `virtualStoreDir`, and the
|
||||
/// install keeps the lockfile in `node_modules/.pnpm`.
|
||||
#[test]
|
||||
fn store_prune_keeps_every_package_installed_by_copy_in_the_global_virtual_store() {
|
||||
assert_prune_keeps_every_package_installed_by_copy(
|
||||
|workspace| enable_gvs_in_workspace_yaml(workspace, ""),
|
||||
"node_modules/.pnpm/lock.yaml",
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_prune_keeps_every_package_installed_by_copy(
|
||||
configure: impl FnOnce(&Path),
|
||||
installed_lockfile: &str,
|
||||
) {
|
||||
let CommandTempCwd {
|
||||
root: _root, workspace, npmrc_info, ..
|
||||
} = CommandTempCwd::init().add_mocked_registry();
|
||||
let AddMockedRegistry {
|
||||
mock_instance: _mock_instance,
|
||||
store_dir,
|
||||
..
|
||||
} = npmrc_info;
|
||||
write_manifest(&workspace);
|
||||
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
|
||||
configure(&workspace);
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
assert!(
|
||||
workspace.join(installed_lockfile).is_file(),
|
||||
"the install must write {installed_lockfile}",
|
||||
);
|
||||
|
||||
let store_dir = StoreDir::new(store_dir);
|
||||
let keys_before = sorted_index_keys(&store_dir);
|
||||
assert!(keys_before.len() > 1, "the install must index every package it fetched");
|
||||
let blob_paths = blob_paths_of_rows(&store_dir, &keys_before);
|
||||
|
||||
prune(&workspace);
|
||||
|
||||
assert_eq!(sorted_index_keys(&store_dir), keys_before);
|
||||
let missing: Vec<_> = blob_paths
|
||||
.iter()
|
||||
.filter(|path| !path.exists())
|
||||
.collect();
|
||||
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
|
||||
}
|
||||
|
||||
/// The install keys a runtime row by its synthesized manifest as well as
|
||||
/// its integrity, and the resolution lists one archive per platform.
|
||||
#[test]
|
||||
fn store_prune_keeps_a_runtime_installed_by_copy() {
|
||||
let root = tempfile::tempdir().expect("create a temp dir");
|
||||
let workspace = prepare_workspace(&root, "packageImportMethod: copy\n");
|
||||
let mut server = mockito::Server::new();
|
||||
let fixtures = [runtime_fixture(&mut server, "node", "22.0.0", host_platform(), host_arch())];
|
||||
write_runtime_manifest(&workspace, &fixtures);
|
||||
write_runtime_lockfile(&workspace, &fixtures);
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--frozen-lockfile", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let store_dir = StoreDir::new(root.path().join("store"));
|
||||
let keys_before = sorted_index_keys(&store_dir);
|
||||
let runtime_keys: Vec<String> = keys_before
|
||||
.iter()
|
||||
.filter(|key| key.ends_with("\tnode@runtime:22.0.0"))
|
||||
.cloned()
|
||||
.collect();
|
||||
let blob_paths = blob_paths_of_rows(&store_dir, &runtime_keys);
|
||||
assert!(!blob_paths.is_empty(), "the install must index the runtime");
|
||||
|
||||
prune(&workspace);
|
||||
|
||||
assert_eq!(sorted_index_keys(&store_dir), keys_before);
|
||||
let missing: Vec<_> = blob_paths
|
||||
.iter()
|
||||
.filter(|path| !path.exists())
|
||||
.collect();
|
||||
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed the runtime's files");
|
||||
}
|
||||
|
||||
/// A lockfile written by a pnpm that uses another lockfile format, such as
|
||||
/// pnpm 8.
|
||||
const LOCKFILE_V6: &str = "lockfileVersion: '6.0'
|
||||
|
||||
dependencies:
|
||||
is-positive:
|
||||
specifier: 1.0.0
|
||||
version: 1.0.0
|
||||
|
||||
packages:
|
||||
|
||||
/is-positive@1.0.0:
|
||||
resolution: {integrity: sha512-xxzPGZ4P2uN6rROUa5N9Z7zTX6ERuE0hs6GUOc/cKBLF2NqKc16UwqHMt3tFg4CO6EBTE5UecUasg+3jZx3Ckg==}
|
||||
dev: false
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn store_prune_skips_an_installed_lockfile_of_an_unsupported_version_with_a_warning() {
|
||||
let CommandTempCwd {
|
||||
root: _root, workspace, npmrc_info, ..
|
||||
} = CommandTempCwd::init().add_mocked_registry();
|
||||
let AddMockedRegistry { mock_instance: _mock_instance, .. } = npmrc_info;
|
||||
write_manifest(&workspace);
|
||||
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
fs::write(workspace.join("node_modules/.pnpm/lock.yaml"), LOCKFILE_V6)
|
||||
.expect("write the old lockfile");
|
||||
|
||||
let output = pacquet_at(&workspace)
|
||||
.with_args(["store", "prune"])
|
||||
.output()
|
||||
.expect("run store prune");
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
eprintln!("STDERR:\n{stderr}\n");
|
||||
assert!(output.status.success(), "store prune failed: {stderr}");
|
||||
let expected = format!(
|
||||
"Skipped {} because it was installed with lockfileVersion 6.0, \
|
||||
which this version of pnpm does not support",
|
||||
dunce::canonicalize(&workspace).expect("canonicalize the workspace").display(),
|
||||
);
|
||||
let warnings: Vec<_> = stderr
|
||||
.lines()
|
||||
.filter(|line| line.contains("lockfileVersion"))
|
||||
.collect();
|
||||
assert_eq!(warnings, [expected.as_str()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_fails_on_a_broken_installed_lockfile_and_removes_nothing() {
|
||||
assert_prune_fails_on_a_broken_file_and_removes_nothing(
|
||||
"node_modules/.pnpm/lock.yaml",
|
||||
"packages: [",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_fails_on_a_broken_modules_manifest_and_removes_nothing() {
|
||||
assert_prune_fails_on_a_broken_file_and_removes_nothing("node_modules/.modules.yaml", "{");
|
||||
}
|
||||
|
||||
fn assert_prune_fails_on_a_broken_file_and_removes_nothing(file: &str, content: &str) {
|
||||
let CommandTempCwd {
|
||||
root: _root, workspace, npmrc_info, ..
|
||||
} = CommandTempCwd::init().add_mocked_registry();
|
||||
let AddMockedRegistry {
|
||||
mock_instance: _mock_instance,
|
||||
store_dir,
|
||||
..
|
||||
} = npmrc_info;
|
||||
write_manifest(&workspace);
|
||||
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
fs::write(workspace.join(file), content).expect("break the file");
|
||||
let store_dir = StoreDir::new(store_dir);
|
||||
let keys_before = sorted_index_keys(&store_dir);
|
||||
|
||||
let output = pacquet_at(&workspace)
|
||||
.with_args(["store", "prune"])
|
||||
.output()
|
||||
.expect("run store prune");
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(!output.status.success(), "store prune must fail: {stderr}");
|
||||
assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}");
|
||||
assert_eq!(sorted_index_keys(&store_dir), keys_before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_prune_fails_on_an_undecodable_row_of_an_installed_package_and_removes_nothing() {
|
||||
let CommandTempCwd {
|
||||
root: _root, workspace, npmrc_info, ..
|
||||
} = CommandTempCwd::init().add_mocked_registry();
|
||||
let AddMockedRegistry {
|
||||
mock_instance: _mock_instance,
|
||||
store_dir,
|
||||
..
|
||||
} = npmrc_info;
|
||||
write_manifest(&workspace);
|
||||
append_workspace_yaml(&workspace, "packageImportMethod: copy\n");
|
||||
pacquet_at(&workspace)
|
||||
.with_args(["install", "--reporter=silent"])
|
||||
.assert()
|
||||
.success();
|
||||
let store_dir = StoreDir::new(store_dir);
|
||||
let keys_before = sorted_index_keys(&store_dir);
|
||||
let blob_paths = blob_paths_of_rows(&store_dir, &keys_before);
|
||||
Command::new("node")
|
||||
.with_args([
|
||||
"-e",
|
||||
r"
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const db = new DatabaseSync(process.argv[1]);
|
||||
db.prepare(`UPDATE package_index SET data = ? WHERE key LIKE '%' || char(9) || 'is-positive@1.0.0'`)
|
||||
.run(Buffer.from([0xc1]));
|
||||
db.close();
|
||||
",
|
||||
])
|
||||
.with_arg(store_dir.root().join("index.db"))
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let output = pacquet_at(&workspace)
|
||||
.with_args(["store", "prune"])
|
||||
.output()
|
||||
.expect("run store prune");
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(!output.status.success(), "store prune must fail: {stderr}");
|
||||
assert!(stderr.contains("ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES"), "stderr={stderr}");
|
||||
assert_eq!(sorted_index_keys(&store_dir), keys_before);
|
||||
let missing: Vec<_> = blob_paths
|
||||
.iter()
|
||||
.filter(|path| !path.exists())
|
||||
.collect();
|
||||
assert_eq!(missing, Vec::<&PathBuf>::new(), "store prune removed files the project uses");
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
pub use runtime::{host_platform_selector, runtime_platform_selector};
|
||||
pub use runtime::{binary_store_index_key, host_platform_selector, runtime_platform_selector};
|
||||
pub use tarball_resolution::{
|
||||
local_file_tarball_install_url, tarball_url_and_integrity, unverified_fetch_is_allowed,
|
||||
};
|
||||
|
||||
@@ -367,6 +367,16 @@ impl BinaryArchiveFetch<'_> {
|
||||
.map_err(InstallPackageBySnapshotError::DownloadTarball)
|
||||
}
|
||||
}
|
||||
/// The store index key of the row that a runtime install writes for
|
||||
/// `binary`, which includes its synthesized manifest.
|
||||
pub fn binary_store_index_key(
|
||||
package_key: &PackageKey,
|
||||
binary: &BinaryResolution,
|
||||
) -> Result<String, InstallPackageBySnapshotError> {
|
||||
let manifest = synthesize_runtime_manifest_bytes(package_key, binary)?;
|
||||
Ok(pnpm_tarball::ArchiveStoreProjection::Package { append_manifest: Some(&manifest) }
|
||||
.store_index_key(&binary.integrity.to_string(), &package_key.pkg_id()))
|
||||
}
|
||||
/// Serialize the synthesized runtime `package.json` to bytes.
|
||||
///
|
||||
/// `serde_json::to_vec` writes a single-line UTF-8 blob. The bytes go
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::{
|
||||
EnvLockfile, Lockfile, ProjectSnapshot, extract_env_document, extract_main_document,
|
||||
ComVer, EnvLockfile, Lockfile, LockfileVersion, ProjectSnapshot, extract_env_document,
|
||||
extract_main_document,
|
||||
git_merge_file::{MERGE_CONFLICT_OURS, ParsedWantedFile, parse_wanted_file},
|
||||
load_lockfile::repair_document::prepare_value_for_fix,
|
||||
merge_lockfile_changes,
|
||||
@@ -7,6 +8,7 @@ use crate::{
|
||||
use derive_more::{Display, Error};
|
||||
use pipe_trait::Pipe;
|
||||
use pnpm_diagnostics::miette::{self, Diagnostic};
|
||||
use serde::Deserialize;
|
||||
use serde_saphyr::MessageFormatter;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
@@ -134,6 +136,26 @@ impl Lockfile {
|
||||
Self::load_from_path(&file_path)
|
||||
}
|
||||
|
||||
/// The `lockfileVersion` of the lockfile at `file_path` when
|
||||
/// [`LockfileVersion`] rejects it, which makes every load of that file
|
||||
/// fail. `None` when the file is absent or its version is accepted or
|
||||
/// unreadable.
|
||||
pub fn unsupported_version_at(file_path: &Path) -> Result<Option<ComVer>, LoadLockfileError> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct VersionField {
|
||||
lockfile_version: ComVer,
|
||||
}
|
||||
let Some(content) = read_lockfile_text(file_path)? else { return Ok(None) };
|
||||
let main = extract_main_document(&content);
|
||||
let Ok(VersionField { lockfile_version }) =
|
||||
serde_saphyr::from_str_with_options(&main, yaml_parse_options(main.len()))
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok((!LockfileVersion::<9>::is_compatible(lockfile_version)).then_some(lockfile_version))
|
||||
}
|
||||
|
||||
/// Load the wanted lockfile (`<dir>/pnpm-lock.yaml`) — a
|
||||
/// directory-addressed loader for callers that resolve into a
|
||||
/// directory other than the process's current one. Returns
|
||||
|
||||
@@ -628,3 +628,22 @@ fn parses_link_dep_in_injected_snapshot() {
|
||||
assert_eq!(c_ref, &SnapshotDepRef::Link("packages/c".to_string()));
|
||||
assert_eq!(c_ref.resolve(&c_name), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_version_at_names_only_a_rejected_version() {
|
||||
let tmp = tempdir().expect("create tempdir");
|
||||
let path = tmp.path().join(Lockfile::CURRENT_FILE_NAME);
|
||||
let version_at = |content: Option<&str>| {
|
||||
if let Some(content) = content {
|
||||
std::fs::write(&path, content).expect("write lock.yaml");
|
||||
}
|
||||
Lockfile::unsupported_version_at(&path)
|
||||
.expect("read lock.yaml")
|
||||
.map(|version| version.to_string())
|
||||
};
|
||||
|
||||
assert_eq!(version_at(None), None);
|
||||
assert_eq!(version_at(Some("lockfileVersion: '6.0'\npackages: {}\n")), Some("6.0".to_string()));
|
||||
assert_eq!(version_at(Some(MAIN_DOC)), None);
|
||||
assert_eq!(version_at(Some("packages: [")), None);
|
||||
}
|
||||
@@ -39,6 +39,9 @@ pub enum PruneError {
|
||||
#[display("Failed to read CAS loader references: {_0}")]
|
||||
#[diagnostic(code(ERR_PNPM_STORE_LOADER_REFERENCES))]
|
||||
LoaderReferences(#[error(source)] io::Error),
|
||||
#[display("Failed to read the store files that registered projects use: {_0}")]
|
||||
#[diagnostic(code(ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES))]
|
||||
InUseFiles(#[error(source)] io::Error),
|
||||
#[diagnostic(transparent)]
|
||||
StoreLock(#[error(source)] StoreLockError),
|
||||
|
||||
@@ -109,9 +112,23 @@ impl StoreDir {
|
||||
///
|
||||
/// [#344]: https://github.com/pnpm/pacquet/issues/344
|
||||
pub fn prune(&self, cache_dir: &Path, protected_dirs: &[&Path]) -> Result<(), PruneError> {
|
||||
self.prune_keeping(cache_dir, protected_dirs, || Ok(HashSet::new()))
|
||||
}
|
||||
|
||||
/// [`Self::prune`], keeping every content-addressable file that
|
||||
/// `in_use_files` returns whatever its hard-link count. `in_use_files`
|
||||
/// runs while the prune holds the store lock, and its error stops the
|
||||
/// prune before it removes anything.
|
||||
pub fn prune_keeping(
|
||||
&self,
|
||||
cache_dir: &Path,
|
||||
protected_dirs: &[&Path],
|
||||
in_use_files: impl FnOnce() -> Result<HashSet<PathBuf>, PruneError>,
|
||||
) -> Result<(), PruneError> {
|
||||
let _store_lock = self.lock_for_prune().map_err(PruneError::StoreLock)?;
|
||||
let references = crate::loader_references::loader_references(self)
|
||||
let mut references = crate::loader_references::loader_references(self)
|
||||
.map_err(PruneError::LoaderReferences)?;
|
||||
references.files.extend(in_use_files()?);
|
||||
self.prune_global_virtual_store(&references.package_roots)?;
|
||||
self.remove_cached_metadata(cache_dir, protected_dirs)?;
|
||||
let stats = crate::prune_cas::prune_cas_with_references(self, &references.files)
|
||||
|
||||
@@ -265,3 +265,40 @@ fn loaded_project_with_nonhidden_local_store_does_not_mark_unused_store_slots()
|
||||
assert!(!dead_child.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_keeping_keeps_in_use_files_and_their_rows() {
|
||||
let root = tempdir().unwrap();
|
||||
let store = StoreDir::new(root.path().join("store"));
|
||||
let digest = |head: &str| format!("{head}{}", "a".repeat(126));
|
||||
let in_use = store.file_path_by_hex_str(&digest("01"), "");
|
||||
let unused = store.file_path_by_hex_str(&digest("02"), "");
|
||||
let index = crate::StoreIndex::open_in(&store).unwrap();
|
||||
for (key, path, digest) in
|
||||
[("in-use", &in_use, digest("01")), ("unused", &unused, digest("02"))]
|
||||
{
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, "{}").unwrap();
|
||||
let file = crate::CafsFileInfo { digest, mode: 0o644, size: 2, checked_at: None };
|
||||
let row = crate::PackageFilesIndex {
|
||||
files: std::iter::once(("package.json".to_string(), file)).collect(),
|
||||
..crate::PackageFilesIndex::default()
|
||||
};
|
||||
index.set(key, &row).unwrap();
|
||||
}
|
||||
drop(index);
|
||||
|
||||
store
|
||||
.prune_keeping(&store.root().join("cache"), &[], || {
|
||||
Ok(std::iter::once(in_use.clone()).collect())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(in_use.exists());
|
||||
assert!(!unused.exists());
|
||||
let keys = crate::StoreIndex::open_in(&store)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.unwrap();
|
||||
assert_eq!(keys, ["in-use"]);
|
||||
}
|
||||
@@ -7,15 +7,17 @@ import { prepareEmpty, preparePackages } from '@pnpm/prepare'
|
||||
import type { ProjectManifest, ProjectRootDir } from '@pnpm/types'
|
||||
import { writeYamlFileSync } from 'write-yaml-file'
|
||||
|
||||
import { testDefaults } from '../utils/index.js'
|
||||
|
||||
const gitUtils = await import('@pnpm/network.git-utils')
|
||||
jest.unstable_mockModule('@pnpm/network.git-utils', () => ({
|
||||
...gitUtils,
|
||||
getCurrentBranch: jest.fn(),
|
||||
getBranchesContainingHead: jest.fn(() => Promise.resolve([])),
|
||||
}))
|
||||
|
||||
const { getCurrentBranch, getBranchesContainingHead } = await import('@pnpm/network.git-utils')
|
||||
const { install, mutateModules } = await import('@pnpm/installing.deps-installer')
|
||||
// The test store loads `@pnpm/store.controller`, which reads lockfiles through `@pnpm/network.git-utils`.
|
||||
const { testDefaults } = await import('../utils/index.js')
|
||||
|
||||
test('install with git-branch-lockfile = true', async () => {
|
||||
prepareEmpty()
|
||||
|
||||
@@ -2,6 +2,7 @@ import { PnpmError } from '@pnpm/error'
|
||||
|
||||
export class LockfileBreakingChangeError extends PnpmError {
|
||||
public filename: string
|
||||
public lockfileVersion?: string
|
||||
constructor (
|
||||
filename: string,
|
||||
opts?: {
|
||||
@@ -11,6 +12,7 @@ export class LockfileBreakingChangeError extends PnpmError {
|
||||
) {
|
||||
super('LOCKFILE_BREAKING_CHANGE', formatMessage(filename, opts))
|
||||
this.filename = filename
|
||||
this.lockfileVersion = opts?.lockfileVersion
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,8 +6,10 @@ import { assertStore } from '@pnpm/assert-store'
|
||||
import { STORE_VERSION } from '@pnpm/constants'
|
||||
import { dlx } from '@pnpm/exec.commands'
|
||||
import { prepare, prepareEmpty } from '@pnpm/prepare'
|
||||
import { getFilePathByModeInCafs, type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { store } from '@pnpm/store.commands'
|
||||
import { REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock'
|
||||
import { StoreIndex } from '@pnpm/store.index'
|
||||
import { getIntegrity, REGISTRY_MOCK_PORT } from '@pnpm/testing.registry-mock'
|
||||
import { rimrafSync } from '@zkochan/rimraf'
|
||||
import { safeExeca as execa } from 'execa'
|
||||
|
||||
@@ -787,3 +789,237 @@ test('prune keeps the lockfile verification log', async () => {
|
||||
|
||||
expect(fs.existsSync(verificationLog)).toBeTruthy()
|
||||
})
|
||||
|
||||
describe('prune keeps the packages that a registered project still uses', () => {
|
||||
// The patch adds a file, so pnpm caches it as a side-effects entry of the row.
|
||||
const MARKER_PATCH = [
|
||||
'diff --git a/patched-marker.txt b/patched-marker.txt',
|
||||
'new file mode 100644',
|
||||
'index 0000000..3f2e1d4',
|
||||
'--- /dev/null',
|
||||
'+++ b/patched-marker.txt',
|
||||
'@@ -0,0 +1 @@',
|
||||
'+patched',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
function prepareProject (workspaceYaml: string): void {
|
||||
prepare({
|
||||
dependencies: {
|
||||
'@pnpm.e2e/pkg-with-1-dep': '100.0.0',
|
||||
'is-positive': '1.0.0',
|
||||
},
|
||||
})
|
||||
fs.writeFileSync('pnpm-workspace.yaml', workspaceYaml)
|
||||
}
|
||||
|
||||
// The store is outside the project, so the install registers the project.
|
||||
async function install (): Promise<string> {
|
||||
const storeDir = path.resolve('..', 'store')
|
||||
await execa('node', [pnpmBin, 'install', `--store-dir=${storeDir}`, `--cache-dir=${path.resolve('..', 'cache')}`, `--registry=${REGISTRY}`])
|
||||
return path.join(storeDir, STORE_VERSION)
|
||||
}
|
||||
|
||||
async function prune (storeDir: string, reporter?: (logObj: unknown) => void): Promise<void> {
|
||||
await store.handler({
|
||||
cacheDir: path.resolve('..', 'cache'),
|
||||
dir: process.cwd(),
|
||||
pnpmHomeDir: '',
|
||||
configByUri: {},
|
||||
registriesByScope: { default: REGISTRY },
|
||||
reporter,
|
||||
storeDir,
|
||||
dlxCacheMaxAge: Infinity,
|
||||
virtualStoreDirMaxLength: process.platform === 'win32' ? 60 : 120,
|
||||
}, ['prune'])
|
||||
}
|
||||
|
||||
function withStoreIndex<Result> (storeDir: string, read: (storeIndex: StoreIndex) => Result): Result {
|
||||
const storeIndex = new StoreIndex(storeDir)
|
||||
try {
|
||||
return read(storeIndex)
|
||||
} finally {
|
||||
storeIndex.close()
|
||||
}
|
||||
}
|
||||
|
||||
function sortedIndexKeys (storeDir: string): string[] {
|
||||
return withStoreIndex(storeDir, (storeIndex) => Array.from(storeIndex.keys()).sort())
|
||||
}
|
||||
|
||||
// Every store file the rows under `keys` name, with their side effects.
|
||||
function blobPathsOfRows (storeDir: string, keys: string[]): string[] {
|
||||
return withStoreIndex(storeDir, (storeIndex) => keys.flatMap((key) => {
|
||||
const row = storeIndex.get(key) as PackageFilesIndex
|
||||
return [
|
||||
...row.files.values(),
|
||||
...Array.from(row.sideEffects?.values() ?? []).flatMap((diff) => Array.from(diff.added?.values() ?? [])),
|
||||
].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode))
|
||||
}))
|
||||
}
|
||||
|
||||
async function expectPruneKeepsEveryRowAndBlob (storeDir: string): Promise<void> {
|
||||
const keysBefore = sortedIndexKeys(storeDir)
|
||||
expect(keysBefore.length).toBeGreaterThan(1)
|
||||
const blobPaths = blobPathsOfRows(storeDir, keysBefore)
|
||||
|
||||
await prune(storeDir)
|
||||
|
||||
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
|
||||
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
|
||||
}
|
||||
|
||||
test('a patched dependency installed with packageImportMethod: hardlink', async () => {
|
||||
// A patched package gets its own copy of the files even under `hardlink`,
|
||||
// so no store file of it has a hard link outside the store.
|
||||
prepareProject('packageImportMethod: hardlink\npatchedDependencies:\n is-positive@1.0.0: patches/is-positive@1.0.0.patch\n')
|
||||
fs.mkdirSync('patches')
|
||||
fs.writeFileSync('patches/is-positive@1.0.0.patch', MARKER_PATCH)
|
||||
const storeDir = await install()
|
||||
const row = withStoreIndex(storeDir, (storeIndex) => {
|
||||
const key = Array.from(storeIndex.keys()).find((key) => key.endsWith('\tis-positive@1.0.0'))!
|
||||
return storeIndex.get(key) as PackageFilesIndex
|
||||
})
|
||||
const sideEffects = Array.from(row.sideEffects?.entries() ?? [])
|
||||
expect(sideEffects.filter(([key]) => key.includes(';patch='))).toHaveLength(1)
|
||||
const blobPaths = [
|
||||
...row.files.values(),
|
||||
...sideEffects.flatMap(([, diff]) => Array.from(diff.added?.values() ?? [])),
|
||||
].map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode))
|
||||
const keysBefore = sortedIndexKeys(storeDir)
|
||||
|
||||
await prune(storeDir)
|
||||
|
||||
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
|
||||
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
|
||||
})
|
||||
|
||||
test('every package installed with packageImportMethod: copy', async () => {
|
||||
// No store file has a hard link outside the store, so the hard-link
|
||||
// count alone marks every package unused.
|
||||
prepareProject('packageImportMethod: copy\n')
|
||||
const storeDir = await install()
|
||||
|
||||
await expectPruneKeepsEveryRowAndBlob(storeDir)
|
||||
})
|
||||
|
||||
// Without a wanted lockfile, the install writes the installed lockfile into
|
||||
// the custom virtual store directory. Under a global virtual store, the
|
||||
// project records the shared store as its virtualStoreDir and keeps the
|
||||
// installed lockfile in node_modules/.pnpm.
|
||||
test.each([
|
||||
['modulesDir: pnpm_modules\n', 'pnpm_modules/.pnpm/lock.yaml'],
|
||||
['virtualStoreDir: vs\n', 'node_modules/.pnpm/lock.yaml'],
|
||||
['virtualStoreDir: vs\nlockfile: false\n', 'vs/lock.yaml'],
|
||||
['enableGlobalVirtualStore: true\n', 'node_modules/.pnpm/lock.yaml'],
|
||||
])('every package installed with packageImportMethod: copy and %j', async (workspaceYaml, installedLockfile) => {
|
||||
prepareProject(`packageImportMethod: copy\n${workspaceYaml}`)
|
||||
const storeDir = await install()
|
||||
expect(fs.existsSync(installedLockfile)).toBe(true)
|
||||
|
||||
await expectPruneKeepsEveryRowAndBlob(storeDir)
|
||||
})
|
||||
|
||||
test('every package installed with packageImportMethod: copy and an absolute modulesDir', async () => {
|
||||
prepareProject('packageImportMethod: copy\n')
|
||||
const modulesDir = path.resolve('..', 'absolute_modules')
|
||||
fs.appendFileSync('pnpm-workspace.yaml', `modulesDir: ${JSON.stringify(modulesDir)}\n`)
|
||||
const storeDir = await install()
|
||||
expect(fs.existsSync(path.join(modulesDir, '.pnpm', 'lock.yaml'))).toBe(true)
|
||||
|
||||
await expectPruneKeepsEveryRowAndBlob(storeDir)
|
||||
})
|
||||
|
||||
// Each project of a workspace without a shared lockfile is registered on
|
||||
// its own, and the workspace-level modulesDir applies to it.
|
||||
test('every package of a workspace project with its own lockfile and a workspace-level modulesDir', async () => {
|
||||
prepare({ name: 'root' })
|
||||
fs.writeFileSync('pnpm-workspace.yaml', 'packages: [project]\nsharedWorkspaceLockfile: false\nmodulesDir: pnpm_modules\npackageImportMethod: copy\n')
|
||||
fs.mkdirSync('project')
|
||||
fs.writeFileSync('project/package.json', JSON.stringify({
|
||||
name: 'project',
|
||||
dependencies: { '@pnpm.e2e/pkg-with-1-dep': '100.0.0', 'is-positive': '1.0.0' },
|
||||
}))
|
||||
const storeDir = await install()
|
||||
expect(fs.existsSync('project/pnpm_modules/.pnpm/lock.yaml')).toBe(true)
|
||||
|
||||
await expectPruneKeepsEveryRowAndBlob(storeDir)
|
||||
})
|
||||
|
||||
test('a runtime installed with packageImportMethod: copy', async () => {
|
||||
// A registry tarball stands in for the runtime archive, so the install
|
||||
// needs no network. The variant targets this host.
|
||||
prepare({ dependencies: { node: 'runtime:22.0.0' } })
|
||||
fs.writeFileSync('pnpm-workspace.yaml', 'packageImportMethod: copy\n')
|
||||
const target = { os: process.platform, cpu: process.arch }
|
||||
fs.writeFileSync('pnpm-lock.yaml', JSON.stringify({
|
||||
lockfileVersion: '9.0',
|
||||
settings: { autoInstallPeers: true, excludeLinksFromLockfile: false },
|
||||
importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } },
|
||||
packages: {
|
||||
'node@runtime:22.0.0': {
|
||||
hasBin: true,
|
||||
resolution: {
|
||||
type: 'variations',
|
||||
variants: [{
|
||||
targets: [target],
|
||||
resolution: {
|
||||
type: 'binary',
|
||||
archive: 'tarball',
|
||||
url: `${REGISTRY}is-positive/-/is-positive-1.0.0.tgz`,
|
||||
integrity: getIntegrity('is-positive', '1.0.0'),
|
||||
bin: { node: 'index.js' },
|
||||
},
|
||||
}],
|
||||
},
|
||||
version: '22.0.0',
|
||||
},
|
||||
},
|
||||
snapshots: { 'node@runtime:22.0.0': {} },
|
||||
}))
|
||||
const storeDir = await install()
|
||||
const keysBefore = sortedIndexKeys(storeDir)
|
||||
const blobPaths = withStoreIndex(storeDir, (storeIndex) => keysBefore
|
||||
.filter((key) => key.endsWith('\tnode@runtime:22.0.0'))
|
||||
.flatMap((key) => Array.from((storeIndex.get(key) as PackageFilesIndex).files.values()))
|
||||
.map(({ digest, mode }) => getFilePathByModeInCafs(storeDir, digest, mode)))
|
||||
expect(blobPaths.length).toBeGreaterThan(0)
|
||||
|
||||
await prune(storeDir)
|
||||
|
||||
expect(sortedIndexKeys(storeDir)).toStrictEqual(keysBefore)
|
||||
expect(blobPaths.filter((blobPath) => !fs.existsSync(blobPath))).toStrictEqual([])
|
||||
})
|
||||
|
||||
test('a project installed with an unsupported lockfile version is skipped with a warning', async () => {
|
||||
prepareProject('packageImportMethod: copy\n')
|
||||
const storeDir = await install()
|
||||
// A lockfile written by a pnpm that uses another lockfile format, such as pnpm 8.
|
||||
fs.writeFileSync('node_modules/.pnpm/lock.yaml', [
|
||||
"lockfileVersion: '6.0'",
|
||||
'',
|
||||
'dependencies:',
|
||||
' is-positive:',
|
||||
' specifier: 1.0.0',
|
||||
' version: 1.0.0',
|
||||
'',
|
||||
'packages:',
|
||||
'',
|
||||
' /is-positive@1.0.0:',
|
||||
` resolution: {integrity: ${getIntegrity('is-positive', '1.0.0')}}`,
|
||||
' dev: false',
|
||||
'',
|
||||
].join('\n'))
|
||||
const reporter = jest.fn()
|
||||
|
||||
await prune(storeDir, reporter)
|
||||
|
||||
const warnings = reporter.mock.calls
|
||||
.map(([logObj]) => logObj as { level?: string, message?: string })
|
||||
.filter(({ level }) => level === 'warn')
|
||||
.map(({ message }) => message)
|
||||
expect(warnings).toStrictEqual([
|
||||
`Skipped ${process.cwd()} because it was installed with lockfileVersion 6.0, which this version of pnpm does not support`,
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -44,18 +44,25 @@
|
||||
".test": "pn pretest && cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pnpm/constants": "workspace:*",
|
||||
"@pnpm/crypto.hash": "workspace:*",
|
||||
"@pnpm/error": "workspace:*",
|
||||
"@pnpm/fetching.fetcher-base": "workspace:*",
|
||||
"@pnpm/hooks.types": "workspace:*",
|
||||
"@pnpm/installing.modules-yaml": "workspace:*",
|
||||
"@pnpm/installing.package-requester": "workspace:*",
|
||||
"@pnpm/lockfile.fs": "workspace:*",
|
||||
"@pnpm/lockfile.types": "workspace:*",
|
||||
"@pnpm/lockfile.utils": "workspace:*",
|
||||
"@pnpm/resolving.resolver-base": "workspace:*",
|
||||
"@pnpm/store.cafs": "workspace:*",
|
||||
"@pnpm/store.controller-types": "workspace:*",
|
||||
"@pnpm/store.create-cafs-store": "workspace:*",
|
||||
"@pnpm/store.index": "workspace:*",
|
||||
"@pnpm/types": "workspace:*",
|
||||
"@pnpm/workspace.workspace-manifest-reader": "workspace:*",
|
||||
"@zkochan/rimraf": "catalog:",
|
||||
"empathic": "catalog:",
|
||||
"is-subdir": "catalog:",
|
||||
"pretty-bytes": "catalog:",
|
||||
"ramda": "catalog:",
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import path from 'node:path'
|
||||
|
||||
import { LOCKFILE_VERSION, WORKSPACE_MANIFEST_FILENAME } from '@pnpm/constants'
|
||||
import { isError, PnpmError } from '@pnpm/error'
|
||||
import { type Modules, readModulesManifest } from '@pnpm/installing.modules-yaml'
|
||||
import { readCurrentLockfile } from '@pnpm/lockfile.fs'
|
||||
import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
|
||||
import { packageIdFromSnapshot } from '@pnpm/lockfile.utils'
|
||||
import { globalWarn } from '@pnpm/logger'
|
||||
import { getFilePathByModeInCafs, type PackageFileInfo, type PackageFilesIndex } from '@pnpm/store.cafs'
|
||||
import { gitHostedStoreIndexKey, pickStoreIndexKey, type StoreIndex, storeIndexKey } from '@pnpm/store.index'
|
||||
import type { DepPath } from '@pnpm/types'
|
||||
import { readWorkspaceManifestKeys } from '@pnpm/workspace.workspace-manifest-reader'
|
||||
import * as find from 'empathic/find'
|
||||
|
||||
import { getRegisteredProjects } from './projectRegistry.js'
|
||||
|
||||
/**
|
||||
* Every file of the store index rows of the packages in each registered
|
||||
* project's installed lockfile, including the files of every side-effects
|
||||
* entry. A package imported by copy or clone, such as a patched or built
|
||||
* one, has no hard link to the store, so the prune needs this list to keep
|
||||
* it. A project without an installed lockfile, a project whose lockfile
|
||||
* version this pnpm does not support, and a package without a row add
|
||||
* nothing.
|
||||
*/
|
||||
export async function getInUseFiles (storeDir: string, storeIndex: StoreIndex): Promise<Set<string>> {
|
||||
const { keys, binaryKeys } = await registeredProjectsIndexKeys(storeDir)
|
||||
for (const key of manifestIndexKeys(storeIndex, storeDir, binaryKeys)) keys.add(key)
|
||||
const files = new Set<string>()
|
||||
for (const key of keys) {
|
||||
const row = readIndexRow(storeIndex, storeDir, key)
|
||||
if (row == null) continue
|
||||
for (const { digest, mode } of rowFiles(row)) {
|
||||
files.add(path.normalize(getFilePathByModeInCafs(storeDir, digest, mode)))
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
interface IndexKey {
|
||||
key: string
|
||||
binary: boolean
|
||||
}
|
||||
|
||||
async function registeredProjectsIndexKeys (storeDir: string): Promise<{ keys: Set<string>, binaryKeys: Set<string> }> {
|
||||
const keys = new Set<string>()
|
||||
const binaryKeys = new Set<string>()
|
||||
for (const projectDir of await getRegisteredProjects(storeDir)) {
|
||||
// eslint-disable-next-line no-await-in-loop -- hold one project's parsed lockfiles at a time
|
||||
for (const { key, binary } of await projectIndexKeys(projectDir)) {
|
||||
keys.add(key)
|
||||
if (binary) binaryKeys.add(key)
|
||||
}
|
||||
}
|
||||
return { keys, binaryKeys }
|
||||
}
|
||||
|
||||
async function projectIndexKeys (projectDir: string): Promise<IndexKey[]> {
|
||||
return (await readInstalledLockfiles(projectDir)).flatMap(lockfileIndexKeys)
|
||||
}
|
||||
|
||||
function lockfileIndexKeys (lockfile: LockfileObject): IndexKey[] {
|
||||
return Object.entries(lockfile.packages ?? {}).flatMap(([depPath, snapshot]) =>
|
||||
packageIndexKeys(snapshot.resolution, packageIdFromSnapshot(depPath as DepPath, snapshot))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* pnpm 12 shares this store and keys the row of a runtime as
|
||||
* `package-manifest\t<manifest integrity>\t<binary key>`. The lockfile has
|
||||
* no manifest integrity, so this scans the keys, but only when the
|
||||
* lockfiles name a binary.
|
||||
*/
|
||||
function manifestIndexKeys (storeIndex: StoreIndex, storeDir: string, binaryKeys: Set<string>): string[] {
|
||||
if (binaryKeys.size === 0) return []
|
||||
const prefix = 'package-manifest\t'
|
||||
const keys: string[] = []
|
||||
try {
|
||||
for (const key of storeIndex.keys()) {
|
||||
if (!key.startsWith(prefix)) continue
|
||||
const separator = key.indexOf('\t', prefix.length)
|
||||
if (separator !== -1 && binaryKeys.has(key.slice(separator + 1))) keys.push(key)
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
throw inUseFilesError(storeDir, err)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
function readIndexRow (storeIndex: StoreIndex, storeDir: string, key: string): PackageFilesIndex | undefined {
|
||||
try {
|
||||
return storeIndex.get(key) as PackageFilesIndex | undefined
|
||||
} catch (err: unknown) {
|
||||
throw inUseFilesError(storeDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The lockfiles the last install wrote, which record what is installed in
|
||||
* the project.
|
||||
*
|
||||
* The install writes the lockfile to `<modulesDir>/.pnpm`, or to the
|
||||
* `virtualStoreDir` that `<modulesDir>/.modules.yaml` records when the
|
||||
* project has no wanted lockfile. Under a global virtual store, that
|
||||
* setting names the shared store. The manifest does not say which case
|
||||
* applies, so the prune reads both.
|
||||
*/
|
||||
async function readInstalledLockfiles (projectDir: string): Promise<LockfileObject[]> {
|
||||
const modulesDir = path.resolve(projectDir, await readConfiguredModulesDir(projectDir))
|
||||
const defaultDir = path.join(modulesDir, '.pnpm')
|
||||
const modules = await readModulesManifestOfProject(modulesDir)
|
||||
const installStateDirs = new Set([defaultDir, modules?.virtualStoreDir ?? defaultDir])
|
||||
const lockfiles = await Promise.all(Array.from(installStateDirs, async (installStateDir) => readInstalledLockfile(projectDir, installStateDir)))
|
||||
return lockfiles.filter((lockfile) => lockfile != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* The `modulesDir` that the nearest `pnpm-workspace.yaml` at or above the
|
||||
* project sets, or `node_modules`. A `modulesDir` set only on the command
|
||||
* line, in the environment, in the global config, or in `packageConfigs`
|
||||
* is not seen.
|
||||
*/
|
||||
async function readConfiguredModulesDir (projectDir: string): Promise<string> {
|
||||
try {
|
||||
// `findWorkspaceDir` would also honor the workspace directory that the
|
||||
// environment of the prune sets, which names no registered project's
|
||||
// workspace.
|
||||
const workspaceManifest = find.file(WORKSPACE_MANIFEST_FILENAME, { cwd: projectDir })
|
||||
if (workspaceManifest == null) return 'node_modules'
|
||||
const workspaceDir = path.dirname(workspaceManifest)
|
||||
// `PnpmSettings` does not declare `modulesDir`.
|
||||
const manifest = await readWorkspaceManifestKeys(workspaceDir, MODULES_DIR_SETTING) as Record<string, unknown> | undefined
|
||||
const modulesDir = manifest?.modulesDir
|
||||
return typeof modulesDir === 'string' ? modulesDir : 'node_modules'
|
||||
} catch (err: unknown) {
|
||||
throw inUseFilesError(projectDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
const MODULES_DIR_SETTING: ReadonlySet<string> = new Set(['modulesDir'])
|
||||
|
||||
async function readModulesManifestOfProject (modulesDir: string): Promise<Modules | null> {
|
||||
try {
|
||||
return await readModulesManifest(modulesDir)
|
||||
} catch (err: unknown) {
|
||||
throw inUseFilesError(modulesDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A lockfile of a version this pnpm cannot read adds nothing, like a
|
||||
* missing one, so one project installed by another pnpm does not stop the
|
||||
* prune. The hard-link check then decides for that project's files.
|
||||
*/
|
||||
async function readInstalledLockfile (projectDir: string, installStateDir: string): Promise<LockfileObject | null> {
|
||||
try {
|
||||
return await readCurrentLockfile(installStateDir, { ignoreIncompatible: false, wantedVersions: [LOCKFILE_VERSION] })
|
||||
} catch (err: unknown) {
|
||||
const lockfileVersion = unsupportedLockfileVersion(err)
|
||||
if (lockfileVersion != null) {
|
||||
globalWarn(`Skipped ${projectDir} because it was installed with lockfileVersion ${lockfileVersion}, which this version of pnpm does not support`)
|
||||
return null
|
||||
}
|
||||
throw inUseFilesError(installStateDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
function unsupportedLockfileVersion (err: unknown): string | undefined {
|
||||
if (!isError(err) || !('code' in err) || err.code !== 'ERR_PNPM_LOCKFILE_BREAKING_CHANGE') return undefined
|
||||
return (err as { lockfileVersion?: string }).lockfileVersion
|
||||
}
|
||||
|
||||
function inUseFilesError (location: string, err: unknown): PnpmError {
|
||||
return new PnpmError('STORE_DIR_PRUNE_IN_USE_FILES',
|
||||
`Failed to read the store files that registered projects use: ${location}: ${isError(err) ? err.message : String(err)}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* The lockfile does not say which key the install used for a git-hosted
|
||||
* package, which has a key for its contents with and without a build, or for
|
||||
* a variations resolution, whose variant depends on the installing host
|
||||
* and its `supportedArchitectures`.
|
||||
*/
|
||||
function packageIndexKeys (resolution: LockfileResolution, pkgId: string): IndexKey[] {
|
||||
if (!('type' in resolution) || resolution.type == null) {
|
||||
if ('tarball' in resolution) {
|
||||
return [true, false].map((built) => packageKey(pickStoreIndexKey(resolution, pkgId, { built })))
|
||||
}
|
||||
return [packageKey(storeIndexKey(resolution.integrity, pkgId))]
|
||||
}
|
||||
switch (resolution.type) {
|
||||
case 'git':
|
||||
return [true, false].map((built) => packageKey(gitHostedStoreIndexKey(pkgId, { built })))
|
||||
case 'binary':
|
||||
return [{ key: storeIndexKey(resolution.integrity, pkgId), binary: true }]
|
||||
case 'variations':
|
||||
return resolution.variants.flatMap((variant) => packageIndexKeys(variant.resolution, pkgId))
|
||||
default:
|
||||
// A directory has no row, and the content of a custom one is not
|
||||
// known here.
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
function packageKey (key: string): IndexKey {
|
||||
return { key, binary: false }
|
||||
}
|
||||
|
||||
function * rowFiles (row: PackageFilesIndex): Iterable<PackageFileInfo> {
|
||||
yield * row.files.values()
|
||||
for (const sideEffectsDiff of row.sideEffects?.values() ?? []) {
|
||||
if (sideEffectsDiff.added != null) yield * sideEffectsDiff.added.values()
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import type { StoreIndex } from '@pnpm/store.index'
|
||||
import { rimraf } from '@zkochan/rimraf'
|
||||
import prettyBytes from 'pretty-bytes'
|
||||
|
||||
import { getInUseFiles } from './inUseFiles.js'
|
||||
import { pruneGlobalVirtualStore } from './pruneGlobalVirtualStore.js'
|
||||
|
||||
const BIG_ONE = BigInt(1) as unknown
|
||||
@@ -19,6 +20,10 @@ export interface PruneOptions {
|
||||
}
|
||||
|
||||
export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, removeAlienFiles?: boolean): Promise<void> {
|
||||
// 0. Collect the files that registered projects still use, before any
|
||||
// removal, so an unreadable lockfile stops the prune untouched
|
||||
const inUseFiles = await getInUseFiles(storeDir, storeIndex)
|
||||
|
||||
// 1. First, prune the global virtual store
|
||||
// This must happen BEFORE pruning the CAS, because removing packages from
|
||||
// the virtual store will reduce hard link counts on files in the CAS
|
||||
@@ -29,7 +34,7 @@ export async function prune ({ cacheDir, storeDir, storeIndex }: PruneOptions, r
|
||||
await removeStaleSpillDirectories(storeDir)
|
||||
|
||||
// 3. Prune the content-addressable store (CAS)
|
||||
const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles)
|
||||
const removedHashes = await pruneContentAddressableStore(storeDir, removeAlienFiles, inUseFiles)
|
||||
|
||||
// 4. Clean up orphaned package index entries
|
||||
removeOrphanedIndexEntries(storeIndex, removedHashes)
|
||||
@@ -89,14 +94,16 @@ interface CafsPruneStats {
|
||||
fileCounter: number
|
||||
totalSize: number
|
||||
removedHashes: Set<string>
|
||||
inUseFiles: Set<string>
|
||||
}
|
||||
|
||||
async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined): Promise<Set<string>> {
|
||||
async function pruneContentAddressableStore (storeDir: string, removeAlienFiles: boolean | undefined, inUseFiles: Set<string>): Promise<Set<string>> {
|
||||
const cafsDir = path.join(storeDir, 'files')
|
||||
const stats: CafsPruneStats = {
|
||||
fileCounter: 0,
|
||||
totalSize: 0,
|
||||
removedHashes: new Set<string>(),
|
||||
inUseFiles,
|
||||
}
|
||||
const dirs = await getSubdirsSafely(cafsDir)
|
||||
await Promise.all(dirs.map(async (dir) => {
|
||||
@@ -130,7 +137,7 @@ async function pruneCafsEntry ({ dir, fileName, subdir, removeAlienFiles, stats
|
||||
}
|
||||
return
|
||||
}
|
||||
if (stat.nlink === 1 || stat.nlink === BIG_ONE) {
|
||||
if ((stat.nlink === 1 || stat.nlink === BIG_ONE) && !stats.inUseFiles.has(filePath)) {
|
||||
stats.totalSize += stat.size
|
||||
await fs.unlink(filePath)
|
||||
stats.fileCounter++
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { promises as fs } from 'node:fs'
|
||||
import { existsSync, promises as fs } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
|
||||
import { afterEach, expect, jest, test } from '@jest/globals'
|
||||
import { getFilePathByModeInCafs } from '@pnpm/store.cafs'
|
||||
import { StoreIndex } from '@pnpm/store.index'
|
||||
import { temporaryDirectory } from 'tempy'
|
||||
|
||||
import { registerProject } from '../src/storeController/projectRegistry.js'
|
||||
import { prune } from '../src/storeController/prune.js'
|
||||
|
||||
const temporaryDirectories: string[] = []
|
||||
@@ -124,6 +126,93 @@ test('prune propagates unexpected spill directory errors', async () => {
|
||||
await expect(prune(options)).rejects.toBe(error)
|
||||
})
|
||||
|
||||
test.each([
|
||||
['installed lockfile', path.join('node_modules', '.pnpm', 'lock.yaml'), 'packages: ['],
|
||||
['modules manifest', path.join('node_modules', '.modules.yaml'), '{'],
|
||||
])('prune fails before it removes anything when a registered project has a broken %s', async (_name, brokenFile, content) => {
|
||||
const options = createPruneOptions()
|
||||
const projectDir = path.join(options.cacheDir, '..', 'project')
|
||||
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
|
||||
await fs.writeFile(path.join(projectDir, brokenFile), content)
|
||||
await registerProject(options.storeDir, projectDir)
|
||||
const unusedFile = path.join(options.storeDir, 'files', '00', 'unused')
|
||||
await fs.mkdir(path.dirname(unusedFile), { recursive: true })
|
||||
await fs.writeFile(unusedFile, 'unused')
|
||||
|
||||
await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' })
|
||||
expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused')
|
||||
})
|
||||
|
||||
test('prune fails before it removes anything when the store index cannot be read', async () => {
|
||||
const options = createPruneOptions()
|
||||
const projectDir = path.join(options.cacheDir, '..', 'project')
|
||||
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
|
||||
await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), [
|
||||
"lockfileVersion: '9.0'",
|
||||
'importers:',
|
||||
" '.': {}",
|
||||
'packages:',
|
||||
' is-positive@1.0.0:',
|
||||
' resolution: {integrity: sha512-AAAA}',
|
||||
'snapshots:',
|
||||
' is-positive@1.0.0: {}',
|
||||
'',
|
||||
].join('\n'))
|
||||
await registerProject(options.storeDir, projectDir)
|
||||
const unusedFile = path.join(options.storeDir, 'files', '00', 'unused')
|
||||
await fs.mkdir(path.dirname(unusedFile), { recursive: true })
|
||||
await fs.writeFile(unusedFile, 'unused')
|
||||
jest.spyOn(options.storeIndex, 'get').mockImplementation(() => {
|
||||
throw new Error('database disk image is malformed')
|
||||
})
|
||||
|
||||
await expect(prune(options)).rejects.toMatchObject({ code: 'ERR_PNPM_STORE_DIR_PRUNE_IN_USE_FILES' })
|
||||
expect(await fs.readFile(unusedFile, 'utf8')).toBe('unused')
|
||||
})
|
||||
|
||||
test('prune keeps a runtime whose index row is keyed with its manifest integrity', async () => {
|
||||
// pnpm 12 shares this store and keys a runtime row as
|
||||
// `package-manifest\t<manifest integrity>\t<integrity>\t<pkgId>`.
|
||||
const options = createPruneOptions()
|
||||
const projectDir = path.join(options.cacheDir, '..', 'project')
|
||||
await fs.mkdir(path.join(projectDir, 'node_modules', '.pnpm'), { recursive: true })
|
||||
const integrity = 'sha512-BBBB'
|
||||
await fs.writeFile(path.join(projectDir, 'node_modules', '.pnpm', 'lock.yaml'), JSON.stringify({
|
||||
lockfileVersion: '9.0',
|
||||
importers: { '.': { dependencies: { node: { specifier: 'runtime:22.0.0', version: 'runtime:22.0.0' } } } },
|
||||
packages: {
|
||||
'node@runtime:22.0.0': {
|
||||
resolution: {
|
||||
type: 'variations',
|
||||
variants: [{
|
||||
targets: [{ os: 'linux', cpu: 'x64' }],
|
||||
resolution: { type: 'binary', archive: 'tarball', url: 'https://example.com/node.tgz', integrity, bin: { node: 'bin/node' } },
|
||||
}],
|
||||
},
|
||||
version: '22.0.0',
|
||||
},
|
||||
},
|
||||
snapshots: { 'node@runtime:22.0.0': {} },
|
||||
}))
|
||||
await registerProject(options.storeDir, projectDir)
|
||||
const files = new Map([
|
||||
['package.json', { digest: 'aa'.repeat(32), mode: 0o644, size: 2 }],
|
||||
['bin/node', { digest: 'bb'.repeat(32), mode: 0o755, size: 2 }],
|
||||
])
|
||||
const blobPaths = Array.from(files.values(), ({ digest, mode }) => getFilePathByModeInCafs(options.storeDir, digest, mode))
|
||||
await Promise.all(blobPaths.map(async (blobPath) => {
|
||||
await fs.mkdir(path.dirname(blobPath), { recursive: true })
|
||||
await fs.writeFile(blobPath, '{}')
|
||||
}))
|
||||
const key = `package-manifest\tsha512-CCCC\t${integrity}\tnode@runtime:22.0.0`
|
||||
options.storeIndex.set(key, { algo: 'sha512', files })
|
||||
|
||||
await prune(options)
|
||||
|
||||
expect(options.storeIndex.has(key)).toBe(true)
|
||||
expect(blobPaths.filter((blobPath) => !existsSync(blobPath))).toStrictEqual([])
|
||||
})
|
||||
|
||||
function createPruneOptions () {
|
||||
const directory = temporaryDirectory()
|
||||
temporaryDirectories.push(directory)
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
{
|
||||
"path": "../../__utils__/prepare"
|
||||
},
|
||||
{
|
||||
"path": "../../core/constants"
|
||||
},
|
||||
{
|
||||
"path": "../../core/error"
|
||||
},
|
||||
@@ -33,15 +36,30 @@
|
||||
{
|
||||
"path": "../../installing/client"
|
||||
},
|
||||
{
|
||||
"path": "../../installing/modules-yaml"
|
||||
},
|
||||
{
|
||||
"path": "../../installing/package-requester"
|
||||
},
|
||||
{
|
||||
"path": "../../lockfile/fs"
|
||||
},
|
||||
{
|
||||
"path": "../../lockfile/types"
|
||||
},
|
||||
{
|
||||
"path": "../../lockfile/utils"
|
||||
},
|
||||
{
|
||||
"path": "../../resolving/resolver-base"
|
||||
},
|
||||
{
|
||||
"path": "../../worker"
|
||||
},
|
||||
{
|
||||
"path": "../../workspace/workspace-manifest-reader"
|
||||
},
|
||||
{
|
||||
"path": "../cafs"
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user