feat: record the tools that build git-hosted dependencies (lockBuildTools) (#16845)

Stage 1 of RFC 21 (pnpm/rfcs PR 21), behind the experimental
lockBuildTools setting.

The lockfile records the package manager and Node.js that prepare each
git-hosted dependency: `tool:`-keyed packages entries (Prefix::Tool),
reached through a new snapshot field, buildDependencies. Install walks
never follow it. Pruning and the filtering walks keep the tools'
metadata. Tools get no snapshots entries, because a fresh install
materializes every snapshot. settings.lockBuildTools is recorded, so
toggling it makes the lockfile stale.

A pass after the fresh lockfile is built records the tools. It carries
records over from the previous lockfile while the package's resolution
is unchanged. Otherwise it reads the package at its commit through
detect_build_environment, which the prepare uses too, and resolves the
tools with the install's resolver chain and policies, reusing a
satisfying recorded version. The resolver chain outlives resolution
only while the setting is on.

On a cold fetch of a built git package, deps-restorer validates each
record with build_tool_artifact, fetches it by integrity into a private
store dir, and hands it to the git fetcher. The fetcher writes shims
that run the recorded files and puts the recorded Node.js on PATH. The
store-index key of the built package carries a digest of the tools.
Frozen verification checks tool entries against the registry package
they are published as.

package_manager_source moves into pnpm-tool-protocol so the CLI, the
recorder and the verifier share the package manager -> npm package
mapping. Package-manager detection and the build check work over a
PackageFiles trait, so a git host's archive can be read from the store's
file map.
This commit is contained in:
Zoltan Kochan authored and GitHub committed 2026-10-11 23:33:08 +02:00
1 parent 8c47795e10
commit 94f60ac4b6
108 files changed
+3261 -286

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
---
"pacquet": minor
---
Added the experimental `lockBuildTools` setting. With it, the lockfile records the package manager and Node.js that build each git-hosted dependency, and every install builds the dependency with exactly those tools, verified against the recorded integrity [RFC 21](https://github.com/pnpm/rfcs/pull/21).
Generated
+2
View File
@@ -5522,6 +5522,7 @@ dependencies = [
"pnpm-package-name",
"pnpm-reporter",
"pnpm-resolving-resolver-base",
"pnpm-tool-protocol",
"pnpm-wasm-host",
"pretty_assertions",
"serde",
@@ -6465,6 +6466,7 @@ version = "0.0.1"
name = "pnpm-tool-protocol"
version = "0.0.1"
dependencies = [
"node-semver",
"pretty_assertions",
]
+4 -1
View File
@@ -203,7 +203,10 @@ fn metadata_store_index_keys(pkg_id: &str, metadata: &PackageMetadata) -> Vec<St
}
fn git_store_index_keys(pkg_id: &str) -> Vec<String> {
vec![git_hosted_store_index_key(pkg_id, true), git_hosted_store_index_key(pkg_id, false)]
vec![
git_hosted_store_index_key(pkg_id, true, None),
git_hosted_store_index_key(pkg_id, false, None),
]
}
fn fallback_pkg_ids(alias: &str, requested_bare: Option<&str>) -> Vec<String> {
+2 -1
View File
@@ -287,10 +287,11 @@ pub(super) fn prune_deploy_lockfile_graph(
let include_optional = dependency_groups.contains(&DependencyGroup::Optional);
let reachable = reachable_deploy_snapshots(importer, snapshots, include_optional, peer_edges);
let reachable_metadata = reachable
let mut reachable_metadata = reachable
.iter()
.map(PackageKey::without_peer)
.collect::<HashSet<_>>();
reachable_metadata.extend(lockfile.build_tool_keys(&reachable));
retain_reachable_snapshots(lockfile, &reachable, include_optional, peer_edges);
if let Some(packages) = lockfile.packages.as_mut() {
packages.retain(|key, _| reachable_metadata.contains(key));
@@ -48,8 +48,9 @@ async fn pnpmfile_checksum(hook: Option<&Arc<dyn PnpmfileHooks>>) -> Option<Stri
}
/// The configured pnpr server, unless the pnpmfile defines a hook that
/// shapes resolution. The server runs no pnpmfile, so such an install
/// resolves locally and warns that the server is not used
/// shapes resolution or `lockBuildTools` is on. The server runs no
/// pnpmfile and records no build tools, so such an install resolves locally
/// and warns that the server is not used
/// ([pnpm/pnpm#14460](https://github.com/pnpm/pnpm/issues/14460)).
pub(crate) async fn pnpr_target<'a, Reporter: self::Reporter>(
state: &'a State,
@@ -58,6 +59,13 @@ pub(crate) async fn pnpr_target<'a, Reporter: self::Reporter>(
let Some(server) = state.config.pnpr_server.as_deref() else {
return Ok(None);
};
if state.config.lock_build_tools {
let server = redact_url_for_display(server);
pnpm_reporter::emit_global_warning::<Reporter>(&format!(
"Resolving dependencies locally because the pnpr server at {server} cannot record build tools (lockBuildTools)",
));
return Ok(None);
}
let pnpmfile_hook = load_pnpmfile(state, pnpr_lockfile_dir(state, link))?;
if let Some(hook) = pnpmfile_hook.as_ref()
&& let Some(unsupported) = resolution_hook(hook.as_ref()).await?
+10 -3
View File
@@ -89,12 +89,19 @@ fn packages_to_check(
.iter()
.flatten()
.filter(|(key, _)| !skipped.contains(key.to_string().as_str()))
.filter_map(|(key, _)| {
.filter_map(|(key, snapshot)| {
let metadata = lockfile.packages
.as_ref()?
.get(&key.without_peer())?;
let store_index_key =
store_index_key_for_resolution(&metadata.resolution, &key.pkg_id(), true)?;
let build_tools = config.lock_build_tools
.then(|| pnpm_lockfile::build_tools_identity(snapshot, lockfile.packages.as_ref()))
.flatten();
let store_index_key = store_index_key_for_resolution(
&metadata.resolution,
&key.pkg_id(),
true,
build_tools.as_deref(),
)?;
let modules_dir = virtual_store_dir
.join(key.to_virtual_store_name(max_length))
.join("node_modules");
+6 -33
View File
@@ -9,11 +9,7 @@
use crate::cli_args::self_update::install_pnpm::pnpm_package_to_install;
use pnpm_env_installer::pnpm_engine_packages;
/// Yarn moved its CLI to `@yarnpkg/cli-dist` in 2.0, and Yarn 6
/// (`yarnpkg/zpm`) is a native binary with no npm package at all.
const YARN_BERRY_MAJOR: u64 = 2;
const YARN_NATIVE_MAJOR: u64 = 6;
use pnpm_tool_protocol::{PackageManagerSource, package_manager_source};
const NPM_PACKAGES: [&str; 1] = ["npm"];
const YARN_CLASSIC_PACKAGES: [&str; 1] = ["yarn"];
@@ -103,11 +99,10 @@ impl PackageManager {
}
pub(crate) fn channel(self, version_spec: &str) -> Channel {
match self {
PackageManager::Bun => Channel::Binary(BinaryChannel::Bun),
PackageManager::Npm => Channel::Registry { package: NPM_PACKAGES[0] },
PackageManager::Pnpm => Channel::Registry { package: "pnpm" },
PackageManager::Yarn => yarn_channel(version_spec),
match (self, package_manager_source(self.name(), version_spec)) {
(_, Some(PackageManagerSource::Registry(package))) => Channel::Registry { package },
(PackageManager::Yarn, _) => Channel::Binary(BinaryChannel::Yarn),
_ => Channel::Binary(BinaryChannel::Bun),
}
}
@@ -126,7 +121,7 @@ impl PackageManager {
links_native_binary: package.links_native_binary,
})
}
PackageManager::Yarn => match yarn_channel(version) {
PackageManager::Yarn => match self.channel(version) {
Channel::Registry { package } if package == YARN_CLASSIC_PACKAGES[0] => {
Some(single_package(&YARN_CLASSIC_PACKAGES))
}
@@ -141,27 +136,5 @@ fn single_package(packages: &'static [&'static str; 1]) -> EnginePackages {
EnginePackages { wrapper: packages[0], pinned: packages, links_native_binary: false }
}
fn yarn_channel(version_spec: &str) -> Channel {
match committed_major(version_spec) {
Some(major) if major >= YARN_NATIVE_MAJOR => Channel::Binary(BinaryChannel::Yarn),
Some(major) if major < YARN_BERRY_MAJOR => {
Channel::Registry { package: YARN_CLASSIC_PACKAGES[0] }
}
_ => Channel::Registry { package: YARN_BERRY_PACKAGES[0] },
}
}
/// The lowest major a specifier admits, or `None` when it admits any
/// version — a dist-tag, a wildcard, or something unparsable. Yarn's
/// channels are split by major, so a specifier that does not commit to one
/// falls through to the current line rather than to Yarn 1.
fn committed_major(version_spec: &str) -> Option<u64> {
let version_spec = version_spec.trim();
if version_spec.is_empty() || version_spec == "*" || version_spec.eq_ignore_ascii_case("x") {
return None;
}
Some(node_semver::Range::parse(version_spec).ok()?.min_version()?.major)
}
#[cfg(test)]
mod tests;
@@ -1108,3 +1108,4 @@ fn assert_packages_prepared_from_their_own_checkout(workspace: &Path, commit: &s
}
mod builds;
mod locked_build_tools;
@@ -120,7 +120,7 @@ fn prepared_git_package_in_shared_store_still_requires_project_approval() {
assert!(!workspace_b.join("node_modules/shared-prepare/prepare.txt").exists());
let store_dir = pnpm_store_dir::StoreDir::from(npmrc_info.store_dir.clone());
let store_index_key = pnpm_store_dir::git_hosted_store_index_key(&spec, true);
let store_index_key = pnpm_store_dir::git_hosted_store_index_key(&spec, true, None);
let store_index = pnpm_store_dir::StoreIndex::open_in(&store_dir).expect("open store index");
let mut legacy_index = store_index
.get(&store_index_key)
@@ -0,0 +1,344 @@
//! `lockBuildTools`: the lockfile records the tools that prepare a
//! git-hosted dependency, and later installs prepare it with exactly those.
//!
//! The package manager is Yarn Classic from the mocked registry, which
//! proxies the real bytes: pnpm verifies a package manager it provisions.
use super::{
CommandTempCwd, GitRepoFixture, Lockfile, LockfileResolution, allow_builds,
append_workspace_yaml_key, assert_eq, assert_success, fs, pnpm_at, read_lockfile,
write_dependencies,
};
use pnpm_lockfile::PackageKey;
use std::path::Path;
/// A repository that pins Yarn 1.22.22 and records, from its `prepare`
/// script, the user agent it ran under and the `node` that ran it.
fn pins_yarn_repo(root: &Path, extra_manifest: &str) -> (GitRepoFixture, String) {
let repo = GitRepoFixture::init(root, "pins-yarn");
repo.write_file(
"package.json",
&format!(
r#"{{"name":"pins-yarn","version":"1.0.0","main":"index.js","packageManager":"yarn@1.22.22","scripts":{{"prepare":"node record.js"}}{extra_manifest}}}"#,
),
);
repo.write_file("index.js", "module.exports = 'ok'\n");
repo.write_file(
"record.js",
"require('fs').writeFileSync('prepared-by.txt', process.env.npm_config_user_agent || '')\n",
);
repo.write_file("yarn.lock", "# yarn lockfile v1\n");
let commit = repo.commit("init");
let spec = repo.git_url_at(&commit);
(repo, spec)
}
/// `pnpm install` with every directory a provisioned package manager could
/// land in pinned under `root`.
fn install(workspace: &Path, root: &Path, args: &[&str]) -> std::process::Output {
pnpm_at(workspace)
.arg("install")
.args(args)
.env("PNPM_HOME", root.join("pnpm-home"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache-home"))
.env("YARN_IGNORE_PATH", "1")
.output()
.expect("run pnpm install")
}
fn prepared_by(workspace: &Path) -> String {
fs::read_to_string(workspace.join("node_modules/pins-yarn/prepared-by.txt"))
.expect("the dependency's prepare script should have run")
}
fn snapshot_key(spec: &str) -> PackageKey {
format!("pins-yarn@{spec}").parse().expect("parse the snapshot key")
}
fn tool_key() -> PackageKey {
"yarn@tool:1.22.22".parse().expect("parse the tool key")
}
fn set_up(root: &Path, workspace: &Path, extra_manifest: &str) -> String {
let (_repo, spec) = pins_yarn_repo(root, extra_manifest);
write_dependencies(workspace, &[("pins-yarn", &spec)]);
allow_builds(workspace, &[&format!("pins-yarn@{spec}")]);
append_workspace_yaml_key(workspace, "lockBuildTools", true);
spec
}
#[test]
fn the_package_manager_a_git_dependency_pins_is_recorded_and_provisioned() {
let CommandTempCwd { root, workspace, npmrc_info, .. } =
CommandTempCwd::init().add_mocked_registry();
let spec = set_up(root.path(), &workspace, "");
let output = install(&workspace, root.path(), &[]);
dbg!(&output);
assert_success(&output);
assert!(prepared_by(&workspace).starts_with("yarn/1.22.22"));
let lockfile: Lockfile = read_lockfile(&workspace.join("pnpm-lock.yaml"));
assert_eq!(
lockfile.settings.as_ref().and_then(|settings| settings.lock_build_tools),
Some(true),
);
let snapshots = lockfile.snapshots.as_ref().expect("snapshots");
let tools: Vec<_> = snapshots[&snapshot_key(&spec)]
.build_dependency_keys()
.map(|(alias, key)| (alias.to_string(), key.to_string()))
.collect();
assert_eq!(tools, [("yarn".to_string(), "yarn@tool:1.22.22".to_string())]);
let LockfileResolution::Tarball(tarball) =
&lockfile.packages.as_ref().expect("packages")[&tool_key()].resolution
else {
panic!("Yarn Classic is recorded as a tarball");
};
assert!(tarball.tarball.ends_with("/yarn/-/yarn-1.22.22.tgz"), "{}", tarball.tarball);
assert!(tarball.integrity.is_some());
// The recorded Yarn ran, not one `pnpm dlx` resolved again.
assert!(
!root
.path()
.join("pnpm-home/package-manager-store")
.exists(),
);
// A build tool is never installed into the project.
assert_eq!(virtual_store_slots(&workspace, "yarn@"), Vec::<String>::new());
assert!(!workspace.join("node_modules/yarn").exists());
// A frozen install into an empty store prepares the dependency again,
// from the record alone.
fs::remove_dir_all(workspace.join("node_modules")).expect("remove node_modules");
let fresh_store = root.path().join("fresh-store");
let store_arg = format!("--store-dir={}", fresh_store.display());
let output = install(&workspace, root.path(), &["--frozen-lockfile", &store_arg]);
dbg!(&output);
assert_success(&output);
assert!(prepared_by(&workspace).starts_with("yarn/1.22.22"));
assert_eq!(read_lockfile(&workspace.join("pnpm-lock.yaml")), lockfile);
drop((root, npmrc_info));
}
/// The virtual-store slots whose names start with `prefix`.
fn virtual_store_slots(workspace: &Path, prefix: &str) -> Vec<String> {
fs::read_dir(workspace.join("node_modules/.pnpm"))
.expect("read the virtual store")
.map(|entry| {
entry
.expect("read a slot")
.file_name()
.to_string_lossy()
.into_owned()
})
.filter(|name| name.starts_with(prefix))
.collect()
}
/// A lockfile that points a recorded tool at a file of its own is refused
/// before anything runs: a tool is only ever its own published artifact.
#[test]
fn a_recorded_tool_must_be_its_own_artifact() {
let CommandTempCwd { root, workspace, npmrc_info, .. } =
CommandTempCwd::init().add_mocked_registry();
set_up(root.path(), &workspace, "");
assert_success(&install(&workspace, root.path(), &[]));
let lockfile_path = workspace.join("pnpm-lock.yaml");
let mut lockfile = read_lockfile(&lockfile_path);
let packages = lockfile.packages.as_mut().expect("packages");
let LockfileResolution::Tarball(tarball) = &mut packages
.get_mut(&tool_key())
.expect("tool")
.resolution
else {
panic!("Yarn Classic is recorded as a tarball");
};
tarball.tarball = "file:./not-yarn.tgz".to_string();
tarball.integrity = None;
fs::write(&lockfile_path, lockfile.to_yaml_string().expect("serialize"))
.expect("write lockfile");
fs::remove_dir_all(workspace.join("node_modules")).expect("remove node_modules");
let fresh_store = root.path().join("fresh-store");
let store_arg = format!("--store-dir={}", fresh_store.display());
let output = install(&workspace, root.path(), &["--frozen-lockfile", &store_arg]);
dbg!(&output);
assert!(!output.status.success());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("ERR_PNPM_RESOLUTION_SHAPE_MISMATCH"), "{stderr}");
assert!(!workspace.join("node_modules/pins-yarn/prepared-by.txt").exists());
drop((root, npmrc_info));
}
#[test]
fn a_recorded_tool_whose_bytes_do_not_match_is_not_run() {
let CommandTempCwd { root, workspace, npmrc_info, .. } =
CommandTempCwd::init().add_mocked_registry();
set_up(root.path(), &workspace, "");
assert_success(&install(&workspace, root.path(), &[]));
let lockfile_path = workspace.join("pnpm-lock.yaml");
let mut lockfile = read_lockfile(&lockfile_path);
let packages = lockfile.packages.as_mut().expect("packages");
let LockfileResolution::Tarball(tarball) = &mut packages
.get_mut(&tool_key())
.expect("tool")
.resolution
else {
panic!("Yarn Classic is recorded as a tarball");
};
// The integrity of another package entirely.
tarball.integrity = Some(
"sha512-qkS8GZYTlbRZk+VYLiR8aaVPxgUkqMf/sGWbdPD9cNHdtx7dGMH9Gy7JTvZY0H7YyR9vxpqjOmPBgxoY2pslOw=="
.parse()
.expect("parse integrity"),
);
fs::write(&lockfile_path, lockfile.to_yaml_string().expect("serialize"))
.expect("write lockfile");
fs::remove_dir_all(workspace.join("node_modules")).expect("remove node_modules");
let fresh_store = root.path().join("fresh-store");
let store_arg = format!("--store-dir={}", fresh_store.display());
append_workspace_yaml_key(&workspace, "fetchRetries", 0);
let output = install(&workspace, root.path(), &["--frozen-lockfile", &store_arg]);
dbg!(&output);
assert!(!output.status.success());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("ERR_PNPM_TARBALL_INTEGRITY"), "{stderr}");
assert!(!workspace.join("node_modules/pins-yarn/prepared-by.txt").exists());
drop((root, npmrc_info));
}
/// Turning the setting on makes a lockfile without records stale, so a
/// frozen install cannot resolve the tools live.
#[test]
fn opting_in_needs_a_lockfile_with_the_records() {
let CommandTempCwd { root, workspace, npmrc_info, .. } =
CommandTempCwd::init().add_mocked_registry();
let (_repo, spec) = pins_yarn_repo(root.path(), "");
write_dependencies(&workspace, &[("pins-yarn", &spec)]);
allow_builds(&workspace, &[&format!("pins-yarn@{spec}")]);
assert_success(&install(&workspace, root.path(), &["--lockfile-only"]));
append_workspace_yaml_key(&workspace, "lockBuildTools", true);
let output = install(&workspace, root.path(), &["--frozen-lockfile"]);
dbg!(&output);
assert!(!output.status.success());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("ERR_PNPM_LOCKFILE_CONFIG_MISMATCH"), "{stderr}");
assert!(stderr.contains("lockBuildTools"), "{stderr}");
assert_success(&install(&workspace, root.path(), &[]));
let lockfile = read_lockfile(&workspace.join("pnpm-lock.yaml"));
assert!(
lockfile.packages
.as_ref()
.expect("packages")
.contains_key(&tool_key()),
);
drop((root, npmrc_info));
}
/// A dependency that pins its development Node.js with `onFail: download`
/// is prepared on that Node.js. The mirror serves a `node` that marks its
/// use and hands over to the host's.
#[test]
#[cfg_attr(windows, ignore = "the mock Node.js is a POSIX shell script")]
fn the_node_a_git_dependency_pins_is_recorded_and_runs_the_build() {
let CommandTempCwd { root, workspace, npmrc_info, .. } =
CommandTempCwd::init().add_mocked_registry();
let version = "24.0.0-rc.4";
let marker = root.path().join("provisioned-node-ran");
let mut server = mockito::Server::new();
let _mocks = mock_node_release(&mut server, version, &marker);
append_workspace_yaml_key(
&workspace,
"nodeDownloadMirrors",
format!("{{ rc: '{}/' }}", server.url()),
);
let spec = set_up(
root.path(),
&workspace,
&format!(
r#","devEngines":{{"runtime":{{"name":"node","version":"{version}","onFail":"download"}}}}"#,
),
);
let output = install(&workspace, root.path(), &[]);
dbg!(&output);
assert_success(&output);
assert!(marker.exists(), "the recorded Node.js should have run the build");
let lockfile = read_lockfile(&workspace.join("pnpm-lock.yaml"));
let tools: Vec<_> = lockfile.snapshots.as_ref().expect("snapshots")[&snapshot_key(&spec)]
.build_dependency_keys()
.map(|(alias, key)| (alias.to_string(), key.to_string()))
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.collect();
assert_eq!(
tools,
[
("node".to_string(), format!("node@tool:{version}")),
("yarn".to_string(), "yarn@tool:1.22.22".to_string()),
],
);
assert!(!workspace.join("node_modules/node").exists());
drop((root, npmrc_info, server));
}
/// Serve one Node.js release whose `bin/node` touches `marker` and then
/// runs the host's `node`.
fn mock_node_release(
server: &mut mockito::Server,
version: &str,
marker: &Path,
) -> Vec<mockito::Mock> {
use sha2::Digest as _;
use std::io::Write as _;
let host_node = which::which("node").expect("the tests need Node.js");
let script =
format!("#!/bin/sh\n: > '{}'\nexec '{}' \"$@\"\n", marker.display(), host_node.display());
let platform = pnpm_graph_hasher::host_platform();
let arch = pnpm_graph_hasher::host_arch();
let archive_name = format!("node-v{version}-{platform}-{arch}.tar.gz");
let mut tar = tar::Builder::new(Vec::new());
let mut header = tar::Header::new_gnu();
header.set_size(script.len() as u64);
header.set_mode(0o755);
tar.append_data(
&mut header,
format!("node-v{version}-{platform}-{arch}/bin/node"),
script.as_bytes(),
)
.expect("append bin/node");
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder
.write_all(&tar.into_inner().expect("finish the archive"))
.expect("compress");
let archive = encoder.finish().expect("finish compressing");
let digest = format!("{:x}", sha2::Sha256::digest(&archive));
vec![
server
.mock("GET", "/index.json")
.with_body(format!(r#"[{{"version":"v{version}","lts":false}}]"#))
.create(),
server
.mock("GET", format!("/v{version}/SHASUMS256.txt").as_str())
.with_body(format!("{digest} {archive_name}\n"))
.create(),
server
.mock("GET", format!("/v{version}/{archive_name}").as_str())
.with_body(archive)
.create(),
]
}
+1
View File
@@ -371,6 +371,7 @@ fn store_status_ignores_skipped_optional_packages() {
&metadata.resolution,
&snapshot_key.pkg_id(),
true,
None,
)
.expect("a registry tarball has a store index key");
+6
View File
@@ -817,6 +817,12 @@ pub struct Config {
/// declares `@types/X`. The `addMissingPeerTypes` setting; default `false`.
pub add_missing_peer_types: bool,
/// When `true`, the package manager and Node.js that prepare a
/// git-hosted dependency are recorded in the lockfile, and every later
/// install provisions exactly the recorded, integrity-checked tools.
/// The experimental `lockBuildTools` setting; default `false`.
pub lock_build_tools: bool,
/// When enabled, dependencies of the root workspace project are used to resolve peer
/// dependencies of any projects in the workspace. It is a useful feature as you can install
/// your peer dependencies only in the root of the workspace, and you can be sure that all
+1
View File
@@ -239,6 +239,7 @@ macro_rules! identically_named_settings {
dedupe_peer_dependents, dedupe_peers, auto_dedupe,
dedupe_direct_deps, dedupe_injected_deps,
strict_peer_dependencies, ignore_compatibility_db, add_missing_peer_types,
lock_build_tools,
resolve_peers_from_workspace_root, verify_store_integrity,
strict_store_pkg_content_check, frozen_store,
include_workspace_root, fail_if_no_match,
@@ -367,6 +367,7 @@ pub struct WorkspaceSettings {
pub strict_peer_dependencies: Option<bool>,
pub ignore_compatibility_db: Option<bool>,
pub add_missing_peer_types: Option<bool>,
pub lock_build_tools: Option<bool>,
pub resolve_peers_from_workspace_root: Option<bool>,
pub block_exotic_subdeps: Option<bool>,
pub verify_store_integrity: Option<bool>,
@@ -123,6 +123,7 @@ impl WorkspaceSettings {
self.strict_peer_dependencies = None;
self.ignore_compatibility_db = None;
self.add_missing_peer_types = None;
self.lock_build_tools = None;
self.resolve_peers_from_workspace_root = None;
self.block_exotic_subdeps = None;
self.hoisting_limits = None;
@@ -35,6 +35,7 @@ fn snap(deps: &[(&str, &str)]) -> SnapshotEntry {
.map(|(n, v)| (name(n), SnapshotDepRef::Plain(ver(v))))
.collect();
SnapshotEntry {
build_dependencies: None,
id: None,
dependencies: (!map.is_empty()).then_some(map),
optional_dependencies: None,
@@ -291,11 +291,8 @@ pub(super) fn upload_side_effects_cache(
if !context.cache.write && !publishes_remotely {
return;
}
let Some(files_index_file) = store_index_key_for_resolution(
&metadata.resolution,
&upload.metadata_key.pkg_id(),
!context.scripts.ignore,
) else {
let Some(files_index_file) = files_index_file(context, snapshot_key, upload.metadata_key)
else {
return;
};
if let Err(err) = upload_and_publish(
@@ -312,6 +309,25 @@ pub(super) fn upload_side_effects_cache(
);
}
}
/// The store-index key the package's files were filed under, which keys a
/// built git-hosted package by its recorded build tools.
fn files_index_file(
context: &BuildOneSnapshot<'_>,
snapshot_key: &PackageKey,
metadata_key: &PackageKey,
) -> Option<String> {
let packages = context.graph.packages?;
let build_tools = context.graph.snapshots
.get(snapshot_key)
.and_then(|snapshot| pnpm_lockfile::build_tools_identity(snapshot, Some(packages)));
store_index_key_for_resolution(
&packages.get(metadata_key)?.resolution,
&metadata_key.pkg_id(),
!context.scripts.ignore,
build_tools.as_deref(),
)
}
pub(super) fn upload_and_publish(
context: &BuildOneSnapshot<'_>,
snapshot_key: &PackageKey,
@@ -33,8 +33,8 @@ fn side_effects_key_for_git_hosted_tarball_matches_warm_lookup() {
});
assert_eq!(
store_index_key_for_resolution(&resolution, pkg_id, true),
Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, true)),
store_index_key_for_resolution(&resolution, pkg_id, true, None),
Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, true, None)),
);
}
#[test]
@@ -48,8 +48,8 @@ fn side_effects_key_for_git_resolution_does_not_require_integrity() {
});
assert_eq!(
store_index_key_for_resolution(&resolution, pkg_id, true),
Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, true)),
store_index_key_for_resolution(&resolution, pkg_id, true, None),
Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, true, None)),
);
}
/// If a failed materialization left the slot without its manifest (a
@@ -22,26 +22,61 @@ pub(super) fn derive_cache_keys(
};
let selector = runtime_platform_selector(supported_architectures);
snapshots
.keys()
.map(|snapshot_key| {
.iter()
.map(|(snapshot_key, snapshot)| {
let cache_key =
snapshot_cache_key(snapshot_key, packages, config.ignore_scripts, &selector)
.map(|mut key| {
if key.is_git_hosted
&& allow_build_policy.check(&snapshot_key.without_peer().to_string())
== Some(false)
{
key.value = Some(pnpm_store_dir::git_hosted_store_index_key(
&snapshot_key.without_peer().pkg_id(),
false,
));
}
key
.map(|key| {
GitHostedBuild { snapshot_key, snapshot, packages }.rekey(
key,
config,
allow_build_policy,
)
});
(snapshot_key.clone(), cache_key)
})
.collect()
}
/// A git-hosted snapshot, whose store-index key depends on how it is
/// built.
struct GitHostedBuild<'a> {
snapshot_key: &'a PackageKey,
snapshot: &'a SnapshotEntry,
packages: &'a HashMap<PackageKey, PackageMetadata>,
}
impl GitHostedBuild<'_> {
/// Key `key` the way the git fetcher files the package: unbuilt when
/// `allowBuilds` denies it, and under its recorded build tools when it
/// is built.
fn rekey(
&self,
mut key: SnapshotCacheKey,
config: &Config,
allow_build_policy: &crate::AllowBuildPolicy,
) -> SnapshotCacheKey {
if !key.is_git_hosted {
return key;
}
let pkg_id = self.snapshot_key.without_peer().pkg_id();
if allow_build_policy.check(&self.snapshot_key.without_peer().to_string()) == Some(false) {
key.value = Some(pnpm_store_dir::git_hosted_store_index_key(&pkg_id, false, None));
return key;
}
let build_tools = config.lock_build_tools
.then(|| pnpm_lockfile::build_tools_identity(self.snapshot, Some(self.packages)))
.flatten();
if let (Some(_), Some(build_tools)) = (&key.value, build_tools) {
key.value = Some(pnpm_store_dir::git_hosted_store_index_key(
&pkg_id,
!config.ignore_scripts,
Some(&build_tools),
));
}
key
}
}
/// Sorted + deduplicated so `prefetch_cas_paths` doesn't redo identical
/// SELECT + integrity-check work for peer variants of one package.
/// Derived leniently over the whole lockfile — the superset over what
@@ -122,6 +157,7 @@ pub(super) fn snapshot_cache_key(
&metadata.resolution,
&pkg_id,
!ignore_scripts,
None,
),
is_git_hosted: true,
})
@@ -304,7 +340,7 @@ pub(super) fn tarball_cache_key(
return Ok(SnapshotCacheKey { value: None, is_git_hosted: false });
}
Ok(SnapshotCacheKey {
value: store_index_key_for_resolution(resolution, pkg_id, !ignore_scripts),
value: store_index_key_for_resolution(resolution, pkg_id, !ignore_scripts, None),
is_git_hosted: tarball.is_git_hosted(),
})
}
@@ -386,6 +386,7 @@ impl<'a> CreateVirtualStore<'a> {
// `drain_cold_downloads` so it doesn't serialize
// inside this cooperative task.
defer_link: true,
packages: Some(inputs.wanted.packages),
#[cfg(test)]
link_concurrency_probe: self.link_concurrency_probe,
}
@@ -70,13 +70,14 @@ pub fn materialization_closure(
skipped.contains_optional_excluded(key)
})
});
let reachable_metadata = metadata_walk
let mut reachable_metadata = metadata_walk
.as_ref()
.unwrap_or(&reachable)
.snapshot_keys
.iter()
.map(PackageKey::without_peer)
.collect::<HashSet<_>>();
reachable_metadata.extend(lockfile.build_tool_keys(&reachable.snapshot_keys));
let importers = lockfile.importers
.iter()
.filter(|(id, _)| reachable.importer_ids.contains(*id))
@@ -125,6 +125,7 @@ fn lockfile_with_top_level(marker: &str, minor: u16) -> Lockfile {
Lockfile {
lockfile_version: LockfileVersion::<9>::try_from(ComVer { major: 9, minor }).unwrap(),
settings: Some(LockfileSettings {
lock_build_tools: None,
auto_install_peers: marker == "fresh",
dedupe_peers: Some(marker == "fresh"),
exclude_links_from_lockfile: marker != "fresh",
@@ -3,6 +3,10 @@ pub use tarball_resolution::{
local_file_tarball_install_url, tarball_url_and_integrity, unverified_fetch_is_allowed,
};
pub use build_tools::BuildToolError;
mod build_tools;
mod fetch;
mod runtime;
@@ -59,6 +63,9 @@ pub struct InstallPackageBySnapshot<'a> {
/// once its tarball is in the store. No effect under
/// [`NodeLinker::Hoisted`](pnpm_modules_yaml::NodeLinker::Hoisted), which never writes virtual-store slots.
pub defer_link: bool,
/// The `packages:` of the lockfile being materialized, where the tools
/// a snapshot's `buildDependencies` name are found.
pub packages: Option<&'a HashMap<pnpm_lockfile::PackageKey, pnpm_lockfile::PackageMetadata>>,
#[cfg(test)]
pub link_concurrency_probe:
Option<&'a crate::create_virtual_dir_by_snapshot::tests::LinkConcurrencyProbe>,
@@ -122,6 +129,9 @@ pub enum InstallPackageBySnapshotError {
#[diagnostic(transparent)]
GitFetch(#[error(source)] GitFetcherError),
#[diagnostic(transparent)]
BuildTool(#[error(source)] BuildToolError),
/// Failure from the directory fetcher: walking the source
/// directory of an injected workspace dep, reading its manifest,
/// or running the npm-packlist filter for
@@ -0,0 +1,306 @@
//! Install the tools a git-hosted snapshot's `buildDependencies` record
//! (`lockBuildTools`), so the git fetcher can prepare the package with
//! exactly those.
//!
//! A tool is fetched like any lockfile entry, verified against its recorded
//! integrity, and imported into a private directory of its own: no install
//! links a build tool into `node_modules`.
use super::{
InstallPackageBySnapshot, InstallPackageBySnapshotError, fetch::download_tarball,
runtime::binary_variant_for_host,
};
use crate::{ImportIndexedDirError, ImportIndexedDirOpts, import_indexed_dir};
use derive_more::{Display, Error};
use miette::Diagnostic;
use pnpm_git_fetcher::{BuildTools, LockedBuildTools, ProvisionedTool};
use pnpm_lockfile::{
BuildToolArtifact, LockfileResolution, PackageKey, PackageMetadata, SnapshotEntry,
TarballResolution, build_tool_artifact,
};
use pnpm_reporter::Reporter;
use pnpm_store_dir::{PrivateInstall, PrivateInstallError};
use pnpm_tarball::IngestTarballToStore;
use std::{collections::HashMap, path::PathBuf};
/// The lockfile name of the runtime a package builds on. Every other tool
/// a snapshot records is the package manager that prepares it.
const NODE: &str = "node";
/// Why the build tools a git-hosted snapshot records could not be installed.
#[derive(Debug, Display, Error, Diagnostic)]
pub enum BuildToolError {
#[display(
"The lockfile records {tool_key} as a build tool of {package_key}, but has no packages entry for it."
)]
#[diagnostic(
code(ERR_PNPM_LOCKFILE_MISSING_BUILD_TOOL),
help(r#"Run "pnpm update" for the dependency to record its build tools again."#)
)]
MissingEntry { package_key: String, tool_key: String },
/// The record names something no build tool can be: an entry under
/// another name, a second package manager, or a resolution that is not
/// the tool's own artifact.
#[display(
"The lockfile records {tool_key} as a build tool of {package_key}, which is not a build tool pnpm can run."
)]
#[diagnostic(
code(ERR_PNPM_LOCKFILE_INVALID_BUILD_TOOL),
help(r#"Run "pnpm update" for the dependency to record its build tools again."#)
)]
InvalidRecord { package_key: String, tool_key: String },
#[display("The lockfile records {tool_key} as a build tool, but its download is {installed}.")]
#[diagnostic(
code(ERR_PNPM_BUILD_TOOL_MISMATCH),
help(
"The lockfile entry of the tool may have been tampered with. Check it before you install."
)
)]
NotTheRecordedTool { tool_key: String, installed: String },
#[diagnostic(transparent)]
PrivateInstall(#[error(source)] PrivateInstallError),
#[display("Failed to install the build tool {tool_key}: {source}")]
#[diagnostic(code(ERR_PNPM_BUILD_TOOL_INSTALL))]
Import {
tool_key: String,
#[error(source)]
source: ImportIndexedDirError,
},
#[display("Failed to read the manifest of the build tool {tool_key}: {source}")]
#[diagnostic(code(ERR_PNPM_BUILD_TOOL_INSTALL))]
ReadManifest {
tool_key: String,
#[error(source)]
source: pnpm_package_manifest::PackageManifestError,
},
}
/// The installed tools of one package. The directories they live in go
/// when this is dropped.
pub(super) struct ProvisionedBuildTools {
tools: LockedBuildTools,
_installs: Vec<PrivateInstall>,
}
/// What the git fetcher prepares one package with.
pub(super) enum PackageBuildTools {
/// `lockBuildTools` is off.
Detected,
/// The recorded tools, installed. `None` when the lockfile records none
/// or the package is not built.
Locked(Option<Box<ProvisionedBuildTools>>),
}
impl PackageBuildTools {
pub(super) fn identity(&self) -> Option<&str> {
match self {
PackageBuildTools::Locked(Some(provisioned)) => Some(&provisioned.tools.identity),
PackageBuildTools::Detected | PackageBuildTools::Locked(None) => None,
}
}
pub(super) fn as_build_tools(&self) -> BuildTools<'_> {
match self {
PackageBuildTools::Detected => BuildTools::Detected,
PackageBuildTools::Locked(provisioned) => {
BuildTools::Locked(provisioned.as_ref().map(|provisioned| &provisioned.tools))
}
}
}
}
impl InstallPackageBySnapshot<'_> {
/// The tools the git-hosted package of `snapshot` is prepared with,
/// installed only when the prepare will run.
pub(super) async fn build_tools_for<Reporter: self::Reporter>(
&self,
package_key: &PackageKey,
snapshot: &SnapshotEntry,
built: bool,
) -> Result<PackageBuildTools, InstallPackageBySnapshotError> {
if !self.ctx.config.lock_build_tools {
return Ok(PackageBuildTools::Detected);
}
if !built {
return Ok(PackageBuildTools::Locked(None));
}
let provisioned = self.provision_build_tools::<Reporter>(package_key, snapshot).await?;
Ok(PackageBuildTools::Locked(provisioned.map(Box::new)))
}
/// Install the tools `snapshot` records. `None` when it records none.
async fn provision_build_tools<Reporter: self::Reporter>(
&self,
package_key: &PackageKey,
snapshot: &SnapshotEntry,
) -> Result<Option<ProvisionedBuildTools>, InstallPackageBySnapshotError> {
let Some(identity) = pnpm_lockfile::build_tools_identity(snapshot, self.packages) else {
return Ok(None);
};
let mut tools = LockedBuildTools { identity, ..LockedBuildTools::default() };
let mut installs = Vec::new();
for (alias, tool_key) in snapshot.build_dependency_keys() {
let record = self.build_tool_record(package_key, alias, &tool_key)?;
let slot = if alias.to_string() == NODE {
&mut tools.node
} else {
&mut tools.package_manager
};
if slot.is_some() {
return Err(invalid_record(package_key, &tool_key).into());
}
let (tool, install) = self.install_build_tool::<Reporter>(&tool_key, record).await?;
installs.push(install);
*slot = Some(tool);
}
Ok(Some(ProvisionedBuildTools { tools, _installs: installs }))
}
/// The entry of the tool `alias` names, when it is one: recorded under
/// its own name, and resolved to the artifact that tool can be.
fn build_tool_record<'p>(
&'p self,
package_key: &PackageKey,
alias: &pnpm_lockfile::PkgName,
tool_key: &PackageKey,
) -> Result<(&'p PackageMetadata, BuildToolArtifact<'p>), BuildToolError> {
let metadata = self.packages
.and_then(|packages| packages.get(&tool_key.without_peer()))
.ok_or_else(|| BuildToolError::MissingEntry {
package_key: package_key.to_string(),
tool_key: tool_key.to_string(),
})?;
let artifact = (tool_key.name == *alias)
.then(|| build_tool_artifact(tool_key, &metadata.resolution))
.flatten()
.ok_or_else(|| invalid_record(package_key, tool_key))?;
Ok((metadata, artifact))
}
async fn install_build_tool<Reporter: self::Reporter>(
&self,
tool_key: &PackageKey,
(metadata, artifact): (&PackageMetadata, BuildToolArtifact<'_>),
) -> Result<(ProvisionedTool, PrivateInstall), InstallPackageBySnapshotError> {
let cas_paths = self.fetch_build_tool::<Reporter>(tool_key, metadata, artifact).await?;
let config = self.ctx.config;
let (name, version) = (tool_key.name.to_string(), tool_key.suffix.version().to_string());
let install = config.store_dir
.create_private_install(&format!("{name}-{version}"))
.map_err(BuildToolError::PrivateInstall)?;
import_indexed_dir::<Reporter>(
self.ctx.caches.logged_methods,
config.package_import_method,
install.dir(),
&cas_paths,
ImportIndexedDirOpts::default(),
)
.map_err(|source| BuildToolError::Import { tool_key: tool_key.to_string(), source })?;
check_installed_identity(tool_key, artifact, install.dir())?;
let runs_on_node = matches!(artifact, BuildToolArtifact::Registry { .. });
let tool = ProvisionedTool::from_dir(name, version, install.dir(), runs_on_node)
.map_err(|source| BuildToolError::ReadManifest {
tool_key: tool_key.to_string(),
source,
})?;
Ok((tool, install))
}
/// The tool's files in the store, fetched and checked against the
/// lockfile's integrity when the store does not have them yet.
async fn fetch_build_tool<Reporter: self::Reporter>(
&self,
tool_key: &PackageKey,
metadata: &PackageMetadata,
artifact: BuildToolArtifact<'_>,
) -> Result<HashMap<String, PathBuf>, InstallPackageBySnapshotError> {
match (artifact, &metadata.resolution) {
(BuildToolArtifact::Registry { tarball, .. }, _) => {
self.download_build_tool_tarball::<Reporter>(tool_key, metadata, tarball).await
}
(BuildToolArtifact::Archive, LockfileResolution::Variations(variations)) => {
let binary =
binary_variant_for_host(variations, tool_key, self.runtime_platform_selector)?;
self.fetch_binary::<Reporter>(binary, tool_key).await
}
(BuildToolArtifact::Archive, LockfileResolution::Binary(binary)) => {
self.fetch_binary::<Reporter>(binary, tool_key).await
}
(BuildToolArtifact::Archive, _) => {
unreachable!("build_tool_artifact admits only archive resolutions for an archive")
}
}
}
async fn download_build_tool_tarball<Reporter: self::Reporter>(
&self,
tool_key: &PackageKey,
metadata: &PackageMetadata,
tarball: &TarballResolution,
) -> Result<HashMap<String, PathBuf>, InstallPackageBySnapshotError> {
let package_id = tool_key.pkg_id();
let ingest = self.ingest(metadata, &package_id);
let download = IngestTarballToStore {
package: pnpm_tarball::TarballPackage {
integrity: tarball.integrity.as_ref(),
url: &tarball.tarball,
..ingest.clone().package
},
..ingest
};
download_tarball::<Reporter>(download, None, false).await
.map_err(InstallPackageBySnapshotError::DownloadTarball)
}
}
/// The installed manifest must name the package and version the record
/// stands for: a tarball of some other package does not become the tool by
/// matching the integrity the lockfile records for it.
fn check_installed_identity(
tool_key: &PackageKey,
artifact: BuildToolArtifact<'_>,
dir: &std::path::Path,
) -> Result<(), BuildToolError> {
let expected_name = match artifact {
BuildToolArtifact::Registry { package, .. } => package.to_string(),
BuildToolArtifact::Archive => tool_key.name.to_string(),
};
let expected_version = tool_key.suffix.version().to_string();
let manifest = pnpm_package_manifest::safe_read_package_json_from_dir(dir)
.map_err(|source| BuildToolError::ReadManifest { tool_key: tool_key.to_string(), source })?
.unwrap_or_default();
let field = |name: &str| {
manifest
.get(name)
.and_then(serde_json::Value::as_str)
.unwrap_or("")
};
if field("name") == expected_name && field("version") == expected_version {
return Ok(());
}
Err(BuildToolError::NotTheRecordedTool {
tool_key: tool_key.to_string(),
installed: format!("{}@{}", field("name"), field("version")),
})
}
fn invalid_record(package_key: &PackageKey, tool_key: &PackageKey) -> BuildToolError {
BuildToolError::InvalidRecord {
package_key: package_key.to_string(),
tool_key: tool_key.to_string(),
}
}
impl From<BuildToolError> for InstallPackageBySnapshotError {
fn from(error: BuildToolError) -> Self {
InstallPackageBySnapshotError::BuildTool(error)
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,32 @@
use super::{BuildToolError, check_installed_identity};
use pnpm_lockfile::{BuildToolArtifact, LockfileResolution, PackageKey, build_tool_artifact};
use std::fs;
fn yarn_artifact() -> (PackageKey, LockfileResolution) {
let key: PackageKey = "yarn@tool:4.9.2".parse().expect("parse key");
let resolution = serde_saphyr::from_str(
"{integrity: sha512-mQ==, tarball: https://registry.npmjs.org/@yarnpkg/cli-dist/-/cli-dist-4.9.2.tgz}",
)
.expect("parse resolution");
(key, resolution)
}
#[test]
fn the_installed_tool_is_the_package_the_record_names() {
let (key, resolution) = yarn_artifact();
let artifact = build_tool_artifact(&key, &resolution).expect("a build tool");
assert!(matches!(artifact, BuildToolArtifact::Registry { package: "@yarnpkg/cli-dist", .. }));
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("package.json"), r#"{"name":"@yarnpkg/cli-dist","version":"4.9.2"}"#)
.unwrap();
assert!(check_installed_identity(&key, artifact, dir.path()).is_ok());
fs::write(dir.path().join("package.json"), r#"{"name":"not-yarn","version":"4.9.2"}"#).unwrap();
let error = check_installed_identity(&key, artifact, dir.path()).expect_err("another package");
assert!(matches!(error, BuildToolError::NotTheRecordedTool { .. }), "{error:?}");
fs::write(dir.path().join("package.json"), r#"{"name":"@yarnpkg/cli-dist","version":"4.0.0"}"#)
.unwrap();
assert!(check_installed_identity(&key, artifact, dir.path()).is_err());
}
@@ -33,6 +33,7 @@ pub(super) struct CustomFetched {
}
pub(super) struct SnapshotFetch<'f> {
package_key: &'f PackageKey,
snapshot: &'f SnapshotEntry,
package_id: &'f str,
/// The effective resolution, which a custom fetcher's `delegate`
/// may have replaced.
@@ -54,6 +55,7 @@ pub(super) struct TarballFetch<'a, AllowBuild> {
/// may have replaced.
resolution: &'a LockfileResolution,
package_key: &'a PackageKey,
snapshot: &'a SnapshotEntry,
package_id: &'a str,
allow_build: &'a AllowBuild,
scripts_prepend_node_path: ExecScriptsPrependNodePath,
@@ -164,8 +166,13 @@ impl InstallPackageBySnapshot<'_> {
emit_progress_resolved::<Reporter>(&package_id, self.ctx.requester);
let download = self.ingest(metadata, &package_id);
let custom =
self.custom_fetch::<Reporter>(package_key, metadata, &package_id, &download).await?;
let custom = self.custom_fetch::<Reporter>(
(package_key, snapshot),
metadata,
&package_id,
&download,
)
.await?;
let resolution = custom.resolution.as_ref().unwrap_or(&metadata.resolution);
// Derived from the effective resolution, not the lockfile's: a
// custom fetcher's `delegate` can resolve to a directory, and
@@ -177,6 +184,7 @@ impl InstallPackageBySnapshot<'_> {
None => {
self.fetch_cas_paths::<Reporter>(SnapshotFetch {
package_key,
snapshot,
package_id: &package_id,
resolution,
download: &download,
@@ -197,7 +205,7 @@ impl InstallPackageBySnapshot<'_> {
Ok(InstalledPackage { cas_paths, source_is_mutable, source_exists })
}
fn ingest<'d>(
pub(super) fn ingest<'d>(
&'d self,
metadata: &'d PackageMetadata,
package_id: &'d str,
@@ -317,6 +325,7 @@ impl InstallPackageBySnapshot<'_> {
download: fetch.download,
resolution: fetch.resolution,
package_key: fetch.package_key,
snapshot: fetch.snapshot,
package_id: fetch.package_id,
allow_build,
scripts_prepend_node_path: exec_scripts_prepend_node_path(self.ctx.config),
@@ -9,7 +9,9 @@ use super::{
use crate::{build_modules::exec_scripts_prepend_node_path, custom_fetcher::CustomFetchOutcome};
use pnpm_config::PmOnFail;
use pnpm_git_fetcher::{GitFetchOutput, GitFetcher, GitHostedTarballFetcher};
use pnpm_lockfile::{BinaryResolution, LockfileResolution, PackageKey, PackageMetadata};
use pnpm_lockfile::{
BinaryResolution, LockfileResolution, PackageKey, PackageMetadata, SnapshotEntry,
};
use pnpm_reporter::Reporter;
use pnpm_store_dir::git_hosted_store_index_key;
use pnpm_tarball::IngestTarballToStore;
@@ -18,7 +20,7 @@ use std::{collections::HashMap, path::PathBuf};
impl InstallPackageBySnapshot<'_> {
pub(in super::super) async fn custom_fetch<Reporter: self::Reporter>(
&self,
package_key: &PackageKey,
(package_key, snapshot): (&PackageKey, &SnapshotEntry),
metadata: &PackageMetadata,
package_id: &str,
download: &IngestTarballToStore<'_>,
@@ -47,7 +49,13 @@ impl InstallPackageBySnapshot<'_> {
}
CustomFetchOutcome::Fetched { resolution, tarball } => {
let cas_paths = self.prepare_custom_tarball::<Reporter>(
SnapshotFetch { package_key, package_id, resolution: &resolution, download },
SnapshotFetch {
package_key,
snapshot,
package_id,
resolution: &resolution,
download,
},
tarball.files_map.clone(),
)
.await?;
@@ -70,6 +78,7 @@ impl InstallPackageBySnapshot<'_> {
download: fetch.download,
resolution: fetch.resolution,
package_key: fetch.package_key,
snapshot: fetch.snapshot,
package_id: fetch.package_id,
allow_build: &allow_build,
scripts_prepend_node_path: exec_scripts_prepend_node_path(self.ctx.config),
@@ -108,7 +117,10 @@ impl InstallPackageBySnapshot<'_> {
let config = self.ctx.config;
let built = !config.ignore_scripts
&& allow_build(&fetch.package_key.without_peer().to_string()) != Some(false);
let files_index_file = git_hosted_store_index_key(fetch.package_id, built);
let build_tools =
self.build_tools_for::<Reporter>(fetch.package_key, fetch.snapshot, built).await?;
let files_index_file =
git_hosted_store_index_key(fetch.package_id, built, build_tools.identity());
let package_name = fetch.package_key.name.to_string();
let GitFetchOutput { cas_paths, built: _built } = GitFetcher {
scripts: prepare_script_options(config, exec_scripts_prepend_node_path(config)),
@@ -131,6 +143,7 @@ impl InstallPackageBySnapshot<'_> {
package_id: fetch.package_id,
package_name: &package_name,
requester: self.ctx.requester,
build_tools: build_tools.as_build_tools(),
}
.run::<Reporter>()
.await
@@ -197,12 +210,12 @@ impl InstallPackageBySnapshot<'_> {
cas_paths: HashMap<String, PathBuf>,
) -> Result<HashMap<String, PathBuf>, InstallPackageBySnapshotError> {
let config = self.ctx.config;
let files_index_file = git_hosted_store_index_key(
fetch.package_id,
!config.ignore_scripts
&& (fetch.allow_build)(&fetch.package_key.without_peer().to_string())
!= Some(false),
);
let built = !config.ignore_scripts
&& (fetch.allow_build)(&fetch.package_key.without_peer().to_string()) != Some(false);
let build_tools =
self.build_tools_for::<Reporter>(fetch.package_key, fetch.snapshot, built).await?;
let files_index_file =
git_hosted_store_index_key(fetch.package_id, built, build_tools.identity());
let GitFetchOutput { cas_paths, built: _built } = GitHostedTarballFetcher {
scripts: prepare_script_options(config, fetch.scripts_prepend_node_path),
store: pnpm_git_fetcher::GitStoreContext {
@@ -216,6 +229,7 @@ impl InstallPackageBySnapshot<'_> {
package_id: fetch.package_id,
requester: self.ctx.requester,
build_tools: build_tools.as_build_tools(),
}
.run::<Reporter>()
.await
@@ -122,6 +122,7 @@ async fn run_snapshot_install_with_session(
let snapshot = pnpm_lockfile::SnapshotEntry::default();
super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: None,
@@ -190,6 +190,7 @@ async fn cold_batch_falls_back_when_prefetch_failed() {
let snapshot = pnpm_lockfile::SnapshotEntry::default();
let err = super::super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: None,
@@ -257,6 +257,7 @@ async fn installing_a_runtime_persists_the_synthesized_manifest_into_the_store_i
// Cold install: fetch the fixture, synthesize the manifest, queue the row.
let cold_cas_paths = super::super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: None,
@@ -341,6 +342,7 @@ async fn installing_a_runtime_persists_the_synthesized_manifest_into_the_store_i
let warm_verified = SharedVerifiedFilesCache::default();
let warm_logged = AtomicU8::new(0);
let warm_cas_paths = super::super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: warm_index.as_ref(),
@@ -60,6 +60,7 @@ async fn cold_batch_reuses_in_flight_prefetch_from_mem_cache() {
let snapshot = pnpm_lockfile::SnapshotEntry::default();
let cas_paths = super::super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: None,
@@ -156,6 +157,7 @@ async fn without_mem_cache_skips_coordination_and_downloads() {
let snapshot = pnpm_lockfile::SnapshotEntry::default();
let err = super::super::InstallPackageBySnapshot {
packages: None,
fetching: crate::SnapshotFetchContext {
http_client: &pnpm_network::ThrottledClient::default(),
store_index: None,
@@ -31,6 +31,7 @@ fn create_config(
cache_dir: &Path,
) -> Config {
Config {
lock_build_tools: false,
node_linker_excluded: Vec::new(),
tools: std::collections::BTreeMap::new(),
indexes_by_ecosystem: std::collections::BTreeMap::new(),
+7 -4
View File
@@ -104,11 +104,17 @@ mod shared_side_effects;
pub const NEEDS_BUILD_MARKER: &str = ".pnpm-needs-build";
/// `build_tools` is the [`pnpm_lockfile::build_tools_identity`] of the
/// package's snapshot, which keys a built git-hosted package.
pub fn store_index_key_for_resolution(
resolution: &pnpm_lockfile::LockfileResolution,
pkg_id: &str,
built: bool,
build_tools: Option<&str>,
) -> Option<String> {
if resolution.is_git_hosted() {
return Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, built, build_tools));
}
match resolution {
pnpm_lockfile::LockfileResolution::Tarball(tarball) => {
Some(pnpm_store_dir::pick_store_index_key(
@@ -116,14 +122,11 @@ pub fn store_index_key_for_resolution(
.as_ref()
.map(ToString::to_string)
.as_deref(),
tarball.is_git_hosted(),
false,
pkg_id,
built,
))
}
pnpm_lockfile::LockfileResolution::Git(_) => {
Some(pnpm_store_dir::git_hosted_store_index_key(pkg_id, built))
}
_ => resolution
.integrity()
.map(|integrity| pnpm_store_dir::store_index_key(&integrity.to_string(), pkg_id)),
@@ -0,0 +1,77 @@
//! What a git-hosted package's build needs, decided by its files at the
//! locked commit. Resolution records it in the lockfile and the prepare pass
//! reads the same answer back, so the two agree by construction.
use crate::{PackageFiles, PreferredPm, WantedPm, detect_wanted_pm};
use pnpm_package_manifest::{BINDING_GYP, downloadable_runtime_version};
use serde_json::Value;
/// Scripts to re-run after `<pm>-install`. `prepare` itself runs as part of
/// `<pm>-install` (npm/yarn/pnpm fold it into the install lifecycle), so it
/// is not invoked separately.
///
/// `prepublishOnly` is omitted: neither npm nor Yarn run it for git-hosted
/// deps.
pub(crate) const PREPUBLISH_SCRIPTS: &[&str] = &["prepublish", "prepack", "publish"];
/// The tools a git-hosted package is prepared with.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BuildEnvironment {
pub package_manager: WantedPm,
/// The Node.js range the package pins in `devEngines.runtime` with
/// `onFail: "download"`.
///
/// `None` when it pins none, when Bun prepares it, or when the package
/// ships a `binding.gyp`: an addon compiled by the prepare has to load
/// in the project's runtime, not in the one the package develops on.
pub node: Option<String>,
}
/// The build environment of the package whose own directory holds `package`
/// and whose repository root holds `repository`. `None` when the package
/// needs no build.
#[must_use]
pub fn detect_build_environment(
repository: &(impl PackageFiles + ?Sized),
package: &(impl PackageFiles + ?Sized),
manifest: &Value,
) -> Option<BuildEnvironment> {
if !requires_build(manifest, package) {
return None;
}
let package_manager = detect_wanted_pm(repository, Some(manifest));
let node = (package_manager.pm != PreferredPm::Bun && !package.contains(BINDING_GYP))
.then(|| downloadable_runtime_version(manifest, "devEngines", "node"))
.flatten()
.filter(|range| node_semver::Range::parse(range).is_ok())
.map(str::to_string);
Some(BuildEnvironment { package_manager, node })
}
/// Whether preparing the package runs any of its scripts: a `prepare`, or a
/// prepublish script that has not already produced the package's `main`.
pub(crate) fn requires_build(manifest: &Value, package: &(impl PackageFiles + ?Sized)) -> bool {
let Some(scripts) = manifest.get("scripts").and_then(Value::as_object) else {
return false;
};
let has_script = |name: &str| {
scripts
.get(name)
.and_then(Value::as_str)
.is_some_and(|script| !script.is_empty())
};
if has_script("prepare") {
return true;
}
if !PREPUBLISH_SCRIPTS.iter().any(|name| has_script(name)) {
return false;
}
let main_file = manifest
.get("main")
.and_then(Value::as_str)
.unwrap_or("index.js");
!package.contains(main_file)
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,125 @@
use super::{BuildEnvironment, detect_build_environment, requires_build};
use crate::{PreferredPm, WantedPm};
use pretty_assertions::assert_eq;
use serde_json::json;
use std::fs;
use tempfile::tempdir;
#[test]
fn requires_build_false_for_empty_scripts() {
let dir = tempdir().unwrap();
let manifest = json!({ "name": "x", "version": "0.0.0" });
assert!(!requires_build(&manifest, dir.path()));
}
#[test]
fn requires_build_true_for_non_empty_prepare() {
let dir = tempdir().unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepare": "tsc" },
});
assert!(requires_build(&manifest, dir.path()));
}
#[test]
fn requires_build_false_when_main_exists_and_prepare_absent() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("index.js"), "").unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepublish": "true" },
});
assert!(!requires_build(&manifest, dir.path()));
}
#[test]
fn requires_build_true_when_main_missing_and_prepublish_set() {
let dir = tempdir().unwrap();
// `main` defaults to `index.js`; create no file so it's missing.
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepack": "rollup -c" },
});
assert!(requires_build(&manifest, dir.path()));
}
#[test]
fn no_environment_for_a_package_without_a_build() {
let dir = tempdir().unwrap();
let manifest = json!({ "name": "x", "version": "0.0.0", "packageManager": "yarn@4.9.2" });
assert_eq!(detect_build_environment(dir.path(), dir.path(), &manifest), None);
}
#[test]
fn environment_names_the_pinned_package_manager_and_node() {
let dir = tempdir().unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"packageManager": "yarn@4.9.2",
"devEngines": {
"runtime": [
{ "name": "bun", "version": "1.2.0", "onFail": "download" },
{ "name": "node", "version": "^24.6.0", "onFail": "download" },
{ "name": "node", "version": "22.0.0", "onFail": "download" },
],
},
"scripts": { "prepare": "tsc" },
});
assert_eq!(
detect_build_environment(dir.path(), dir.path(), &manifest),
Some(BuildEnvironment {
package_manager: WantedPm {
pm: PreferredPm::Yarn,
version_spec: Some("4.9.2".to_string()),
pinned: true,
},
node: Some("^24.6.0".to_string()),
}),
);
}
#[test]
fn environment_skips_a_node_pin_that_does_not_ask_for_a_download() {
let dir = tempdir().unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"devEngines": { "runtime": { "name": "node", "version": "24.6.0", "onFail": "error" } },
"scripts": { "prepare": "tsc" },
});
let environment = detect_build_environment(dir.path(), dir.path(), &manifest);
dbg!(&environment);
assert_eq!(environment.and_then(|environment| environment.node), None);
}
/// An addon the prepare compiles has to load in the project's Node.js.
#[test]
fn environment_keeps_a_native_build_on_the_project_runtime() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("binding.gyp"), "{}").unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"devEngines": { "runtime": { "name": "node", "version": "24.6.0", "onFail": "download" } },
"scripts": { "prepare": "node-gyp rebuild" },
});
let environment = detect_build_environment(dir.path(), dir.path(), &manifest);
dbg!(&environment);
assert_eq!(environment.and_then(|environment| environment.node), None);
}
/// The package manager is read from the repository root, the build from
/// the package's own directory.
#[test]
fn environment_reads_the_lockfile_at_the_repository_root() {
let root = tempdir().unwrap();
fs::write(root.path().join("bun.lock"), "{}").unwrap();
let package = root.path().join("packages/x");
fs::create_dir_all(&package).unwrap();
let manifest = json!({ "name": "x", "version": "0.0.0", "scripts": { "prepare": "tsc" } });
let environment = detect_build_environment(root.path(), package.as_path(), &manifest);
dbg!(&environment);
assert_eq!(
environment.map(|environment| environment.package_manager.pm),
Some(PreferredPm::Bun),
);
}
+24
View File
@@ -65,6 +65,30 @@ pub enum PreparePackageError {
source: std::io::Error,
},
/// `lockBuildTools` is on and the lockfile records no package manager
/// for a git-hosted package that has to be built.
#[display(
r#"The lockfile records no build tools for the git-hosted package "{dep_path}", which is prepared with {package_manager}."#
)]
#[diagnostic(
code(ERR_PNPM_LOCKFILE_MISSING_BUILD_TOOLS),
help(
r#"Run "pnpm install" without "--frozen-lockfile" to record them. If a regular install reports this too, run "pnpm update" for the dependency."#
)
)]
MissingBuildTools { dep_path: String, package_manager: String },
/// The package manager the lockfile records for a git-hosted package is
/// not the one the package asks for at the locked commit.
#[display(
r#"The lockfile records {recorded} as the package manager of the git-hosted package "{dep_path}", but the package asks for {package_manager}."#
)]
#[diagnostic(
code(ERR_PNPM_LOCKFILE_BUILD_TOOLS_MISMATCH),
help(r#"Run "pnpm update" for the dependency to record its build tools again."#)
)]
BuildToolMismatch { dep_path: String, recorded: String, package_manager: String },
#[display("I/O error during preparePackage: {_0}")]
#[diagnostic(code(ERR_PNPM_GIT_FETCHER_PREPARE_PACKAGE_IO))]
Io(#[error(source)] std::io::Error),
+26 -1
View File
@@ -65,6 +65,7 @@ pub struct GitFetcher<'a> {
/// `git+…#<commit>` id and carries no name.
pub package_name: &'a str,
pub requester: &'a str,
pub build_tools: crate::BuildTools<'a>,
}
/// Output of [`GitFetcher::run`]. Mirrors the shape of
@@ -128,6 +129,7 @@ impl GitFetcher<'_> {
self.store.files_index_file,
self.package_id,
self.scripts.ignore,
self.build_tools.identity(),
);
queue_files_index(
self.store.index_writer,
@@ -173,6 +175,7 @@ impl<'a> GitFetcher<'a> {
extra_bin_paths: &[],
extra_env: &NO_EXTRA_ENV,
build_tools: self.build_tools,
}
}
}
@@ -388,6 +391,26 @@ pub struct GitManifestQuery<'a> {
pub async fn read_git_manifest(
query: GitManifestQuery<'_>,
) -> Result<Option<Value>, GitFetcherError> {
read_git_package(&query, |_, _, manifest| manifest)
}
/// The [`BuildEnvironment`](crate::BuildEnvironment) of the package a `Git`
/// resolution points at, read from the same checkout as
/// [`read_git_manifest`]. `None` when the package needs no build.
pub async fn read_git_build_environment(
query: GitManifestQuery<'_>,
) -> Result<Option<crate::BuildEnvironment>, GitFetcherError> {
read_git_package(&query, |root, pkg_dir, manifest| {
crate::detect_build_environment(root, pkg_dir, manifest.as_ref()?)
})
}
/// Check the package of `query` out and hand `read` the repository root,
/// the package directory, and its manifest.
fn read_git_package<Output>(
query: &GitManifestQuery<'_>,
read: impl FnOnce(&Path, &Path, Option<Value>) -> Option<Output>,
) -> Result<Option<Output>, GitFetcherError> {
tokio::task::block_in_place(|| {
let source = query.source_cache
.get_with_config(
@@ -408,7 +431,9 @@ pub async fn read_git_manifest(
// happily read an arbitrary `package.json` off the host and
// stamp its name onto this dep.
let pkg_dir = safe_join_path(source.path(), query.path).map_err(GitFetcherError::Prepare)?;
safe_read_package_json_from_dir(&pkg_dir).map_err(GitFetcherError::ReadManifest)
let manifest =
safe_read_package_json_from_dir(&pkg_dir).map_err(GitFetcherError::ReadManifest)?;
Ok(read(source.path(), &pkg_dir, manifest))
})
}
@@ -304,6 +304,7 @@ fn failing_fetcher<'a>(
git_bin: &'a Path,
) -> GitFetcher<'a> {
GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -28,6 +28,7 @@ async fn fetcher_rejects_option_shaped_commit() {
let store_root = tempdir().unwrap();
let store_dir = StoreDir::from(store_root.path().to_path_buf());
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -72,6 +73,7 @@ async fn fetcher_rejects_partial_commit_before_running_git() {
let store_root = tempdir().unwrap();
let store_dir = StoreDir::from(store_root.path().to_path_buf());
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -122,6 +124,7 @@ async fn fetcher_imports_package_into_cas() {
let repo_url = format!("file://{}", bare.display());
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -178,6 +181,7 @@ async fn fetcher_rejects_commit_mismatch() {
// commit.
let bogus = "0000000000000000000000000000000000000000";
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -248,6 +252,7 @@ async fn fetcher_blocks_build_when_not_allowed() {
let store_dir = StoreDir::from(store_root.path().to_path_buf());
let repo_url = format!("file://{}", bare.display());
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -312,6 +317,7 @@ async fn fetcher_runs_prepare_script_when_allowed() {
let repo_url = format!("file://{}", bare.display());
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -371,6 +377,7 @@ async fn fetcher_surfaces_prepare_failure() {
let repo_url = format!("file://{}", bare.display());
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -453,6 +460,7 @@ async fn fetcher_runs_prepare_when_allow_build_returns_true() {
&|dep_path| (dep_path == "x@git+file:///tmp/repo.git#abc123").then_some(true);
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -553,6 +561,7 @@ async fn fetcher_uses_shallow_fetch_for_allowed_hosts() {
env.set("PACQUET_GIT_SHIM_FAKE_COMMIT", fake_commit);
GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -646,6 +655,7 @@ async fn fetcher_clones_when_host_not_in_shallow_list() {
env.set("PACQUET_GIT_SHIM_FAKE_COMMIT", fake_commit);
GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -12,6 +12,7 @@ async fn fetcher_packs_subfolder_when_path_set() {
let repo_url = format!("file://{}", bare.display());
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -90,6 +91,7 @@ async fn fetcher_skips_build_when_ignore_scripts() {
let store_dir = StoreDir::from(store_root.path().to_path_buf());
let repo_url = format!("file://{}", bare.display());
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: true,
unsafe_perm: true,
@@ -14,6 +14,7 @@ async fn fetcher_handles_repo_without_package_json() {
let repo_url = format!("file://{}", bare.display());
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -67,6 +68,7 @@ async fn fetcher_rejects_untrusted_manifest_identity() {
&|dep_path| (!dep_path.contains("://")).then_some(true);
let err = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -129,6 +131,7 @@ async fn fetcher_allows_untrusted_manifest_identity_by_dep_path() {
&|dep_path| (dep_path == "x@git+file:///tmp/repo.git#abc123").then_some(true);
let received = GitFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
+8 -1
View File
@@ -23,13 +23,17 @@ pub(crate) use pnpm_process as process;
#[cfg(not(target_family = "wasm"))]
pub(crate) use std::process;
pub use build_environment::{BuildEnvironment, detect_build_environment};
pub use error::{GitFetcherError, PreparePackageError};
pub use fetcher::{
CheckoutOptions, GitFetchOutput, GitFetcher, GitManifestQuery, cache_checkout_bundles,
checkout_cached_bundles, checkout_commit, checkout_existing_revision, checkout_revision,
checkout_submodules, checkout_submodules_offline, read_git_manifest,
checkout_submodules, checkout_submodules_offline, read_git_build_environment,
read_git_manifest,
};
pub use locked_build_tools::{BuildTools, LockedBuildTools, ProvisionedTool};
pub use options::{GitSource, GitStoreContext, PrepareScriptOptions, RunningPnpm};
pub use package_files::{CasPackageFiles, PackageFiles};
pub use pnpm_fs_packlist::{PacklistError, packlist};
pub use preferred_pm::{PreferredPm, WantedPm, detect_preferred_pm, detect_wanted_pm};
pub use prepare_package::{
@@ -39,9 +43,12 @@ pub use protocols::{SUPPORTED_GIT_PROTOCOLS, read_allowed_git_protocols};
pub use source_cache::GitSourceCache;
pub use tarball_fetcher::GitHostedTarballFetcher;
mod build_environment;
mod cas_io;
mod error;
mod fetcher;
mod locked_build_tools;
mod package_files;
mod pm_shims;
mod preferred_pm;
mod prepare_package;
@@ -0,0 +1,91 @@
//! The tools the lockfile records for building a git-hosted package, which
//! the caller installs before the prepare runs.
use pnpm_cmd_shim::get_bins_from_package_manifest_with_walk;
use pnpm_package_manifest::safe_read_package_json_from_dir;
use std::{
collections::BTreeMap,
path::{Path, PathBuf},
};
/// Where [`crate::prepare_package()`] gets the tools a build runs with.
#[derive(Debug, Default, Clone, Copy)]
pub enum BuildTools<'a> {
/// The package manager the package asks for, provisioned when it pins
/// a version or the host cannot satisfy it, and the host's Node.js.
#[default]
Detected,
/// Only the tools the lockfile records (`lockBuildTools`). `None` when
/// it records none for the package.
Locked(Option<&'a LockedBuildTools>),
}
impl BuildTools<'_> {
/// [`LockedBuildTools::identity`] of the recorded tools, if any.
#[must_use]
pub fn identity(&self) -> Option<&str> {
match self {
BuildTools::Locked(Some(tools)) => Some(&tools.identity),
BuildTools::Detected | BuildTools::Locked(None) => None,
}
}
}
/// The recorded tools of one package, installed.
#[derive(Debug, Default)]
pub struct LockedBuildTools {
/// `None` when pnpm prepares the package: the running pnpm, or the
/// version the package pins, does that as it does without a record.
pub package_manager: Option<ProvisionedTool>,
/// `None` when the package pins no Node.js for its build, which then
/// runs on the host's.
pub node: Option<ProvisionedTool>,
/// Names every recorded tool by lockfile key and resolution. Part of
/// the store-index key of the package the tools build.
pub identity: String,
}
/// A tool installed from its lockfile entry into a directory of its own.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProvisionedTool {
/// The tool's name in the lockfile (`yarn`, `node`).
pub name: String,
pub version: String,
/// The commands the tool publishes, by name.
pub bins: BTreeMap<String, PathBuf>,
/// Whether the commands are scripts Node.js runs rather than
/// executables of their own.
pub runs_on_node: bool,
}
impl ProvisionedTool {
/// Read the commands of the tool installed at `dir` from its
/// `package.json`.
pub fn from_dir(
name: String,
version: String,
dir: &Path,
runs_on_node: bool,
) -> Result<Self, pnpm_package_manifest::PackageManifestError> {
let manifest = safe_read_package_json_from_dir(dir)?.unwrap_or_default();
let bins =
get_bins_from_package_manifest_with_walk(&manifest, dir, |_| Ok(Vec::<PathBuf>::new()))
.into_iter()
.map(|command| (command.name, command.path))
.collect();
Ok(ProvisionedTool { name, version, bins, runs_on_node })
}
/// The command `name`, or the tool's own command when it publishes no
/// such alias (a single-executable Yarn has no `yarnpkg`).
#[must_use]
pub fn command(&self, name: &str) -> Option<&Path> {
self.bins
.get(name)
.or_else(|| self.bins.get(&self.name))
.map(PathBuf::as_path)
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,34 @@
use super::ProvisionedTool;
use pretty_assertions::assert_eq;
use std::fs;
use tempfile::tempdir;
#[test]
fn commands_are_read_from_the_installed_manifest() {
let dir = tempdir().unwrap();
fs::write(
dir.path().join("package.json"),
r#"{"name":"@yarnpkg/cli-dist","bin":{"yarn":"./bin/yarn.js","yarnpkg":"./bin/yarn.js"}}"#,
)
.unwrap();
let tool = ProvisionedTool::from_dir("yarn".to_string(), "4.9.2".to_string(), dir.path(), true)
.expect("read the tool");
assert_eq!(tool.command("yarnpkg"), Some(dir.path().join("bin/yarn.js").as_path()));
assert!(tool.runs_on_node);
}
/// A single-executable Yarn publishes no `yarnpkg`, and a build that calls
/// it still gets the one Yarn.
#[test]
fn a_missing_alias_falls_back_to_the_tool_command() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("package.json"), r#"{"name":"yarn","bin":{"yarn":"yarn"}}"#).unwrap();
let tool =
ProvisionedTool::from_dir("yarn".to_string(), "6.0.0".to_string(), dir.path(), false)
.expect("read the tool");
assert_eq!(tool.command("yarnpkg"), Some(dir.path().join("yarn").as_path()));
}
@@ -0,0 +1,92 @@
//! Read access to the files of a git-hosted package, wherever they are: a
//! checkout on disk while it is prepared, or the store's file map of a git
//! host's archive while it is resolved.
use std::{
collections::HashMap,
fs,
io::Read as _,
path::{Component, Path, PathBuf},
};
/// The files of one directory of a package, addressed by paths relative to
/// it with forward slashes.
pub trait PackageFiles {
/// Whether `path` names a file.
fn contains(&self, path: &str) -> bool;
/// Up to `limit` leading bytes of the file at `path`, or `None` when it
/// cannot be read.
fn read_head(&self, path: &str, limit: u64) -> Option<Vec<u8>>;
}
/// Paths come from the package's own manifest, so one that is absolute or
/// climbs out of the directory names no file of it.
impl PackageFiles for Path {
fn contains(&self, path: &str) -> bool {
is_inside(path) && self.join(path).exists()
}
fn read_head(&self, path: &str, limit: u64) -> Option<Vec<u8>> {
is_inside(path)
.then(|| read_file_head(&self.join(path), limit))
.flatten()
}
}
fn is_inside(path: &str) -> bool {
Path::new(path)
.components()
.all(|component| matches!(component, Component::Normal(_) | Component::CurDir))
}
/// A directory of an archive the store holds, as the map from each file's
/// path in the archive to its content-addressed copy.
pub struct CasPackageFiles<'a> {
pub files: &'a HashMap<String, PathBuf>,
/// The directory within the archive, as a resolution's `path` spells
/// it (`/packages/foo`). `None` is the archive root.
pub dir: Option<&'a str>,
}
impl CasPackageFiles<'_> {
fn archive_path(&self, path: &str) -> Option<String> {
if !is_inside(path) {
return None;
}
let path = path.trim_start_matches("./").trim_end_matches('/');
Some(match self.dir.unwrap_or("").trim_matches('/') {
"" => path.to_string(),
dir => format!("{dir}/{path}"),
})
}
}
impl PackageFiles for CasPackageFiles<'_> {
/// A directory counts too, as it does on disk: `main` may name one.
fn contains(&self, path: &str) -> bool {
let Some(path) = self.archive_path(path) else { return false };
let dir_prefix = format!("{path}/");
self.files.contains_key(&path)
|| self.files
.keys()
.any(|file| file.starts_with(&dir_prefix))
}
fn read_head(&self, path: &str, limit: u64) -> Option<Vec<u8>> {
read_file_head(self.files.get(&self.archive_path(path)?)?, limit)
}
}
fn read_file_head(path: &Path, limit: u64) -> Option<Vec<u8>> {
let mut head = Vec::new();
fs::File::open(path)
.ok()?
.take(limit)
.read_to_end(&mut head)
.ok()?;
Some(head)
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,63 @@
use super::{CasPackageFiles, PackageFiles};
use std::{collections::HashMap, fs, path::PathBuf};
#[test]
fn cas_files_resolve_paths_under_the_package_directory() {
let store = tempfile::tempdir().expect("create store dir");
let lockfile = store.path().join("lockfile-blob");
fs::write(&lockfile, "# yarn lockfile v1\n").expect("write blob");
let files: HashMap<String, PathBuf> = [
("packages/foo/package.json".to_string(), store.path().join("manifest-blob")),
("packages/foo/yarn.lock".to_string(), lockfile),
]
.into_iter()
.collect();
let package = CasPackageFiles { files: &files, dir: Some("/packages/foo") };
assert!(package.contains("package.json"));
assert!(package.contains("./package.json"));
assert!(!package.contains("binding.gyp"));
assert_eq!(package.read_head("yarn.lock", 6).as_deref(), Some(&b"# yarn"[..]));
}
#[test]
fn cas_files_at_the_archive_root() {
let files = HashMap::from([("bun.lock".to_string(), PathBuf::from("/nowhere"))]);
let root = CasPackageFiles { files: &files, dir: None };
assert!(root.contains("bun.lock"));
assert_eq!(root.read_head("bun.lock", 10), None);
}
#[test]
fn cas_files_contain_directories() {
let files = HashMap::from([("pkg/lib/index.js".to_string(), PathBuf::from("/nowhere"))]);
let package = CasPackageFiles { files: &files, dir: Some("pkg") };
assert!(package.contains("lib"));
assert!(package.contains("./lib/"));
assert!(!package.contains("li"));
}
/// A manifest's `main` cannot make the build check look outside the
/// package.
#[test]
fn paths_outside_the_package_name_no_file() {
let root = tempfile::tempdir().unwrap();
let package = root.path().join("package");
fs::create_dir_all(&package).unwrap();
fs::write(root.path().join("outside.js"), "").unwrap();
let files = HashMap::from([("outside.js".to_string(), root.path().join("outside.js"))]);
let archive = CasPackageFiles { files: &files, dir: Some("package") };
let absolute = root.path().join("outside.js");
let absolute = absolute.to_string_lossy();
let package = package.as_path();
assert!(!package.contains("../outside.js"));
assert!(!package.contains(&absolute));
assert_eq!(package.read_head("../outside.js", 1), None);
assert!(!archive.contains("../outside.js"));
if cfg!(windows) {
assert!(!package.contains(r"..\outside.js"));
}
}
+87 -1
View File
@@ -15,7 +15,7 @@ use std::{
path::{Path, PathBuf},
};
use crate::preferred_pm::WantedPm;
use crate::{ProvisionedTool, preferred_pm::WantedPm};
/// The commands a package manager answers to, all of which a dependency's
/// scripts may reach for, and how each one is run.
@@ -71,6 +71,92 @@ pub(crate) fn write_pm_shims(
Ok(written)
}
/// Write shims into `dir` that run each command of `pm` from `tool`, the
/// package manager the lockfile records. A tool whose commands are scripts
/// runs on `node`, or on the host's `node` when it is `None`.
pub(crate) fn write_locked_pm_shims(
dir: &Path,
pm: crate::preferred_pm::PreferredPm,
tool: &ProvisionedTool,
node: Option<&Path>,
) -> io::Result<()> {
fs::create_dir_all(dir)?;
for (name, run_as) in shim_commands(pm) {
let (command, args) = run_as
.split_first()
.unwrap_or((name, &[]));
let target = tool
.command(command)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
format!("{}@{} publishes no {command} command", tool.name, tool.version),
)
})?;
let launcher = match (tool.runs_on_node, node) {
(false, _) => Launcher::Direct,
(true, None) => Launcher::HostNode,
(true, Some(node)) => Launcher::Node(node),
};
let program = LockedProgram { launcher, target, args };
let (file_name, contents) = locked_shim_file(name, &program);
write_executable(&dir.join(file_name), &contents)?;
}
Ok(())
}
/// What a locked shim runs: `target` with `args` in front of the caller's.
struct LockedProgram<'a> {
launcher: Launcher<'a>,
target: &'a Path,
args: &'a [&'a str],
}
/// How a locked shim starts its target.
#[derive(Clone, Copy)]
enum Launcher<'a> {
/// The target is an executable.
Direct,
/// The target is a script for whatever `node` the build's `PATH` has.
HostNode,
/// The target is a script for this Node.js.
Node(&'a Path),
}
#[cfg(any(unix, target_os = "wasi"))]
fn locked_shim_file(name: &str, program: &LockedProgram<'_>) -> (String, String) {
use pnpm_cmd_shim::sh_single_quote;
let mut words: Vec<String> = Vec::new();
match program.launcher {
Launcher::Node(node) => words.push(sh_single_quote(&node.to_string_lossy())),
Launcher::HostNode => words.push("node".to_string()),
Launcher::Direct => {}
}
words.push(sh_single_quote(&program.target.to_string_lossy()));
words.extend(program.args.iter().map(|arg| sh_single_quote(arg)));
(name.to_string(), format!("#!/bin/sh\nexec {} \"$@\"\n", words.join(" ")))
}
#[cfg(windows)]
fn locked_shim_file(name: &str, program: &LockedProgram<'_>) -> (String, String) {
use pnpm_cmd_shim::cmd_escape;
let mut words: Vec<String> = Vec::new();
match program.launcher {
Launcher::Node(node) => words.push(format!(r#""{}""#, cmd_escape(&node.to_string_lossy()))),
Launcher::HostNode => words.push("node".to_string()),
Launcher::Direct => {}
}
words.push(format!(r#""{}""#, cmd_escape(&program.target.to_string_lossy())));
words.extend(
program.args
.iter()
.map(|arg| format!(r#""{}""#, cmd_escape(arg))),
);
(format!("{name}.cmd"), format!("@{} %*\r\n", words.join(" ")))
}
/// Put the running pnpm on the build's `PATH` as `pnpm`, returning the
/// scratch directory its shim lives in.
///
@@ -162,3 +162,86 @@ fn the_running_pnpm_is_called_without_dlx() {
assert!(body.contains("/opt/pnpm"), "{body}");
assert!(!body.contains("dlx"), "{body}");
}
fn provisioned(bins: &[(&str, &str)], runs_on_node: bool) -> crate::ProvisionedTool {
crate::ProvisionedTool {
name: "npm".to_string(),
version: "10.9.2".to_string(),
bins: bins
.iter()
.map(|(name, path)| ((*name).to_string(), std::path::PathBuf::from(path)))
.collect(),
runs_on_node,
}
}
/// A recorded package manager runs from its own directory, on the recorded
/// Node.js, with no `pnpm dlx` resolving it again.
#[test]
fn a_locked_package_manager_runs_its_recorded_files() {
let dir = tempdir().unwrap();
let tool = provisioned(
&[("npm", "/tools/npm/bin/npm-cli.js"), ("npx", "/tools/npm/bin/npx-cli.js")],
true,
);
super::write_locked_pm_shims(
dir.path(),
PreferredPm::Npm,
&tool,
Some(Path::new("/tools/node/bin/node")),
)
.expect("write the shims");
let npm = shim_body(dir.path(), "npm");
assert!(npm.contains("/tools/node/bin/node"), "{npm}");
assert!(npm.contains("/tools/npm/bin/npm-cli.js"), "{npm}");
assert!(!npm.contains("dlx"), "{npm}");
let npx = shim_body(dir.path(), "npx");
assert!(npx.contains("/tools/npm/bin/npx-cli.js"), "{npx}");
}
#[test]
fn a_locked_package_manager_without_a_recorded_node_runs_on_the_hosts() {
let dir = tempdir().unwrap();
let tool = provisioned(
&[("npm", "/tools/npm/bin/npm-cli.js"), ("npx", "/tools/npm/bin/npx-cli.js")],
true,
);
super::write_locked_pm_shims(dir.path(), PreferredPm::Npm, &tool, None)
.expect("write the shims");
let npm = shim_body(dir.path(), "npm");
assert!(npm.contains("node "), "{npm}");
}
/// `bunx` is `bun x` of the recorded executable.
#[test]
fn a_locked_executable_runs_directly() {
let dir = tempdir().unwrap();
let tool = crate::ProvisionedTool {
name: "bun".to_string(),
..provisioned(&[("bun", "/tools/bun/bun")], false)
};
super::write_locked_pm_shims(
dir.path(),
PreferredPm::Bun,
&tool,
Some(Path::new("/tools/node")),
)
.expect("write the shims");
let bunx = shim_body(dir.path(), "bunx");
assert!(bunx.contains("/tools/bun/bun"), "{bunx}");
assert!(!bunx.contains("/tools/node"), "{bunx}");
let subcommand = if cfg!(windows) { r#""x""# } else { "'x'" };
assert!(bunx.contains(subcommand), "{bunx}");
}
#[test]
fn a_locked_tool_without_the_command_fails() {
let dir = tempdir().unwrap();
let tool = provisioned(&[("npx", "/tools/npm/bin/npx-cli.js")], true);
let error = super::write_locked_pm_shims(dir.path(), PreferredPm::Npm, &tool, None)
.expect_err("npm is missing");
assert_eq!(error.kind(), std::io::ErrorKind::NotFound);
}
+19 -20
View File
@@ -1,11 +1,11 @@
//! Detect the package manager and version expected by a git-hosted dependency
//! from the manifest and lockfile or workspace configuration it ships.
use crate::PackageFiles;
use pnpm_package_manifest::package_manager_spec::{
dev_engines_package_managers, engine_name_version, split_spec, version_without_build,
};
use serde_json::Value;
use std::{fs, io::Read as _, path::Path};
/// Package manager a git-hosted dep wants to install with. The variant
/// drives the synthesized `<pm>-install` script in
@@ -54,48 +54,52 @@ impl PreferredPm {
}
}
/// Determine the package manager and version to prepare the package at `dir`.
/// Determine the package manager and version to prepare the package whose
/// repository root holds `files`.
///
/// Manifest pins (`packageManager`, `devEngines.packageManager`) take precedence
/// over lockfiles and `pnpm-workspace.yaml`. When Yarn is selected without an
/// explicit version specifier, the Yarn line is inferred from `yarn.lock`.
#[must_use]
pub fn detect_wanted_pm(dir: &Path, manifest: Option<&Value>) -> WantedPm {
pub fn detect_wanted_pm(
files: &(impl PackageFiles + ?Sized),
manifest: Option<&Value>,
) -> WantedPm {
let wanted = manifest
.and_then(manifest_pin)
.unwrap_or_else(|| WantedPm {
pm: detect_preferred_pm(dir),
pm: detect_preferred_pm(files),
version_spec: None,
pinned: false,
});
if wanted.version_spec.is_some() || wanted.pm != PreferredPm::Yarn {
return wanted;
}
WantedPm { version_spec: yarn_line_of_lockfile(dir), ..wanted }
WantedPm { version_spec: yarn_line_of_lockfile(files), ..wanted }
}
const YARN_CLASSIC_SPEC: &str = "1";
const YARN_BERRY_SPEC: &str = ">=2";
/// Sniff `dir` for a lockfile or `pnpm-workspace.yaml` and return the matching package manager.
/// Sniff `files` for a lockfile or `pnpm-workspace.yaml` and return the matching package manager.
///
/// Lockfiles take precedence over `pnpm-workspace.yaml`. Defaults to [`PreferredPm::Npm`]
/// when neither is present.
#[must_use]
pub fn detect_preferred_pm(dir: &Path) -> PreferredPm {
if dir.join("pnpm-lock.yaml").exists() {
pub fn detect_preferred_pm(files: &(impl PackageFiles + ?Sized)) -> PreferredPm {
if files.contains("pnpm-lock.yaml") {
return PreferredPm::Pnpm;
}
if dir.join("yarn.lock").exists() {
if files.contains("yarn.lock") {
return PreferredPm::Yarn;
}
if dir.join("package-lock.json").exists() {
if files.contains("package-lock.json") {
return PreferredPm::Npm;
}
if dir.join("bun.lockb").exists() || dir.join("bun.lock").exists() {
if files.contains("bun.lockb") || files.contains("bun.lock") {
return PreferredPm::Bun;
}
if dir.join("pnpm-workspace.yaml").exists() {
if files.contains("pnpm-workspace.yaml") {
return PreferredPm::Pnpm;
}
PreferredPm::Npm
@@ -131,16 +135,11 @@ fn pinned_version(version: &str) -> Option<String> {
node_semver::Range::parse(version).is_ok().then(|| version.to_string())
}
/// Detect whether `yarn.lock` in `dir` was written by Yarn Classic or Yarn Berry.
fn yarn_line_of_lockfile(dir: &Path) -> Option<String> {
/// Detect whether the `yarn.lock` among `files` was written by Yarn Classic or Yarn Berry.
fn yarn_line_of_lockfile(files: &(impl PackageFiles + ?Sized)) -> Option<String> {
const HEADER_BYTES: u64 = 64 * 1024;
let lockfile = fs::File::open(dir.join("yarn.lock")).ok()?;
let mut header = Vec::new();
lockfile
.take(HEADER_BYTES)
.read_to_end(&mut header)
.ok()?;
let header = files.read_head("yarn.lock", HEADER_BYTES)?;
let berry = header
.split(|byte| *byte == b'\n')
.any(|line| line.trim_ascii_start().starts_with(b"__metadata:"));
+17 -53
View File
@@ -7,6 +7,8 @@
//! escape the git root via [`safe_join_path`].
use crate::{
BuildTools,
build_environment::{PREPUBLISH_SCRIPTS, requires_build},
error::PreparePackageError,
pm_shims::{provide_running_pnpm, shim_names, write_pm_shims},
preferred_pm::{PreferredPm, WantedPm, detect_wanted_pm},
@@ -25,15 +27,6 @@ use std::{
sync::{LazyLock, Mutex, PoisonError},
};
/// Scripts to re-run after `<pm>-install` finishes. `prepare` itself
/// runs automatically as part of `<pm>-install` (npm/yarn/pnpm fold it
/// into the install lifecycle), so we don't need to invoke it
/// separately.
///
/// Note: `prepublishOnly` is intentionally omitted here — neither npm
/// nor Yarn run it for git-hosted deps.
const PREPUBLISH_SCRIPTS: &[&str] = &["prepublish", "prepack", "publish"];
/// Closure shape used to ask the install policy whether the package at
/// a dep path is allowed to run lifecycle scripts. `Some(false)` explicitly
/// skips preparation; `None` requires a decision before installation.
@@ -56,6 +49,7 @@ pub struct PreparePackageOptions<'a> {
pub pkg_resolution_id: &'a str,
pub extra_bin_paths: &'a [PathBuf],
pub extra_env: &'a HashMap<String, String>,
pub build_tools: BuildTools<'a>,
}
/// Result of [`prepare_package`], distinguishing a package that requires
@@ -73,12 +67,14 @@ impl PreparedPackage {
requested_key: &'a str,
package_id: &str,
ignore_scripts: bool,
build_tools: Option<&str>,
) -> std::borrow::Cow<'a, str> {
if self.should_be_built
&& ((self.ignored_build && !ignore_scripts)
|| (!self.ignored_build && requested_key.ends_with("\tnot-built")))
{
pnpm_store_dir::git_hosted_store_index_key(package_id, !self.ignored_build).into()
pnpm_store_dir::git_hosted_store_index_key(package_id, !self.ignored_build, build_tools)
.into()
} else {
requested_key.into()
}
@@ -97,10 +93,7 @@ pub fn prepare_package<Reporter: self::Reporter>(
let Some(manifest) = manifest else {
return Ok(PreparedPackage { pkg_dir, should_be_built: false, ignored_build: false });
};
let scripts = manifest.get("scripts").and_then(Value::as_object);
if scripts.is_none_or(serde_json::Map::is_empty)
|| !package_should_be_built(&manifest, &pkg_dir)
{
if !requires_build(&manifest, pkg_dir.as_path()) {
return Ok(PreparedPackage { pkg_dir, should_be_built: false, ignored_build: false });
}
if opts.scripts.ignore
@@ -117,17 +110,16 @@ pub fn prepare_package<Reporter: self::Reporter>(
let pm = wanted_pm.pm;
let dep_path = manifest_dep_path(&manifest);
let mut extra_bin_paths = opts.extra_bin_paths.to_vec();
// Kept alive until the prepare is over: dropping it takes the shims
// with it.
let shims_dir =
provide_wanted_pm::<Reporter>(&wanted_pm, &dep_path, opts.scripts.running_pnpm)?;
if let Some(dir) = shims_dir.as_ref() {
extra_bin_paths.insert(0, dir.path().to_path_buf());
}
let build_path = build_path::provide_build_tools::<Reporter>(opts, &wanted_pm, &dep_path)?;
let extra_bin_paths = build_path.bin_paths(opts.extra_bin_paths);
let extra_env = prepare_env(opts.extra_env, opts.scripts.running_pnpm.pm_on_fail);
let run_opts = opts.lifecycle_options(&dep_path, &pkg_dir, &extra_bin_paths, &extra_env);
let mut run_opts = opts.lifecycle_options(&dep_path, &pkg_dir, &extra_bin_paths, &extra_env);
if let Some(node) = build_path.node {
run_opts.environment.node_execpath = Some(node);
}
run_install_and_prepublish::<Reporter>(pm, &run_opts, &manifest)?;
remove_install_node_modules(&pkg_dir)?;
@@ -303,7 +295,7 @@ fn prepublish_invocation(
///
/// A `pmOnFail` other than `download` leaves pnpm itself to the running
/// pnpm, whatever version the dependency pins.
fn provide_wanted_pm<Reporter: self::Reporter>(
pub(super) fn provide_wanted_pm<Reporter: self::Reporter>(
wanted_pm: &WantedPm,
dep_path: &str,
running_pnpm: crate::RunningPnpm<'_>,
@@ -366,7 +358,7 @@ fn provide_package_manager(
Ok(dir)
}
fn describe_wanted_pm(wanted: &WantedPm) -> String {
pub(super) fn describe_wanted_pm(wanted: &WantedPm) -> String {
match &wanted.version_spec {
Some(version_spec) => format!("{}@{version_spec}", wanted.pm.name()),
None => wanted.pm.name().to_string(),
@@ -450,36 +442,6 @@ pub fn resolve_package_build_permission(
})
}
/// Decide whether the package needs building.
fn package_should_be_built(manifest: &Value, pkg_dir: &Path) -> bool {
let Some(scripts) = manifest.get("scripts").and_then(Value::as_object) else {
return false;
};
if scripts
.get("prepare")
.and_then(Value::as_str)
.is_some_and(|script| !script.is_empty())
{
return true;
}
let has_prepublish_script = PREPUBLISH_SCRIPTS
.iter()
.any(|name| {
scripts
.get(*name)
.and_then(Value::as_str)
.is_some_and(|script| !script.is_empty())
});
if !has_prepublish_script {
return false;
}
let main_file = manifest
.get("main")
.and_then(Value::as_str)
.unwrap_or("index.js");
!pkg_dir.join(main_file).exists()
}
/// Join `sub` onto `root` and reject results that climb outside.
///
/// `sub` is a resolution's `path` field, which keeps the leading slash
@@ -521,5 +483,7 @@ fn map_lifecycle_err(source: LifecycleScriptError) -> PreparePackageError {
PreparePackageError::LifecycleFailed { source }
}
mod build_path;
#[cfg(test)]
mod tests;
@@ -0,0 +1,97 @@
//! The tools a prepare runs with, put on its `PATH`.
use super::{PreparePackageOptions, describe_wanted_pm, provide_wanted_pm};
use crate::{
BuildTools, LockedBuildTools,
error::PreparePackageError,
pm_shims::write_locked_pm_shims,
preferred_pm::{PreferredPm, WantedPm},
};
use pnpm_network::redact_and_sanitize;
use pnpm_reporter::Reporter;
use std::path::{Path, PathBuf};
/// What the build's `PATH` gets on top of the caller's: the shims of the
/// package manager, and the directory of the recorded Node.js.
pub(super) struct BuildPath<'a> {
/// Removed when dropped, taking the shims with it.
shims_dir: Option<tempfile::TempDir>,
pub(super) node: Option<&'a Path>,
}
impl BuildPath<'_> {
pub(super) fn bin_paths(&self, extra_bin_paths: &[PathBuf]) -> Vec<PathBuf> {
let shims = self.shims_dir
.as_ref()
.map(|dir| dir.path().to_path_buf());
let node = self.node.and_then(Path::parent).map(Path::to_path_buf);
shims
.into_iter()
.chain(extra_bin_paths.iter().cloned())
.chain(node)
.collect()
}
}
pub(super) fn provide_build_tools<'a, Reporter: self::Reporter>(
opts: &PreparePackageOptions<'a>,
wanted_pm: &WantedPm,
dep_path: &str,
) -> Result<BuildPath<'a>, PreparePackageError> {
let running_pnpm = opts.scripts.running_pnpm;
Ok(match opts.build_tools {
BuildTools::Detected => BuildPath {
shims_dir: provide_wanted_pm::<Reporter>(wanted_pm, dep_path, running_pnpm)?,
node: None,
},
BuildTools::Locked(tools) => BuildPath {
shims_dir: provide_locked_pm::<Reporter>(wanted_pm, tools, dep_path, running_pnpm)?,
node: tools.and_then(locked_node),
},
})
}
/// The recorded Node.js the build runs on, when the lockfile records one.
fn locked_node(tools: &LockedBuildTools) -> Option<&Path> {
tools.node.as_ref()?.command("node")
}
/// [`provide_wanted_pm`] for a package whose build tools the lockfile
/// records: the recorded package manager, and nothing from the host.
///
/// pnpm is the exception. The lockfile records no pnpm, so a package that
/// asks for pnpm is prepared the way [`provide_wanted_pm`] prepares it.
fn provide_locked_pm<Reporter: self::Reporter>(
wanted_pm: &WantedPm,
tools: Option<&LockedBuildTools>,
dep_path: &str,
running_pnpm: crate::RunningPnpm<'_>,
) -> Result<Option<tempfile::TempDir>, PreparePackageError> {
if wanted_pm.pm == PreferredPm::Pnpm {
return provide_wanted_pm::<Reporter>(wanted_pm, dep_path, running_pnpm);
}
let Some(tool) = tools.and_then(|tools| tools.package_manager.as_ref()) else {
return Err(PreparePackageError::MissingBuildTools {
dep_path: redact_and_sanitize(dep_path),
package_manager: describe_wanted_pm(wanted_pm),
});
};
if tool.name != wanted_pm.pm.name() {
return Err(PreparePackageError::BuildToolMismatch {
dep_path: redact_and_sanitize(dep_path),
recorded: format!("{}@{}", tool.name, tool.version),
package_manager: describe_wanted_pm(wanted_pm),
});
}
let node = tools.and_then(locked_node);
tempfile::tempdir()
.and_then(|dir| {
write_locked_pm_shims(dir.path(), wanted_pm.pm, tool, node)?;
Ok(dir)
})
.map(Some)
.map_err(|source| PreparePackageError::PackageManagerUnavailable {
package_manager: format!("{}@{}", tool.name, tool.version),
source,
})
}
@@ -1,7 +1,4 @@
use super::{
PreparePackageOptions, PreparedPackage, package_should_be_built, prepare_package,
safe_join_path,
};
use super::{PreparePackageOptions, PreparedPackage, prepare_package, safe_join_path};
use crate::error::PreparePackageError;
use miette::Diagnostic;
use pnpm_executor::ScriptsPrependNodePath;
@@ -29,6 +26,7 @@ fn write_manifest(dir: &Path, manifest: &serde_json::Value) {
fn opts<'a>(allow: bool, ignore_scripts: bool) -> PreparePackageOptions<'a> {
static EMPTY_BIN_PATHS: &[std::path::PathBuf] = &[];
PreparePackageOptions {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: ignore_scripts,
unsafe_perm: true,
@@ -50,6 +48,7 @@ fn opts<'a>(allow: bool, ignore_scripts: bool) -> PreparePackageOptions<'a> {
fn opts_allow_registry_artifacts_only<'a>() -> PreparePackageOptions<'a> {
static EMPTY_BIN_PATHS: &[std::path::PathBuf] = &[];
PreparePackageOptions {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -74,6 +73,7 @@ fn opts_allow_dep_path<'a>(
) -> PreparePackageOptions<'a> {
static EMPTY_BIN_PATHS: &[std::path::PathBuf] = &[];
PreparePackageOptions {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -92,45 +92,6 @@ fn opts_allow_dep_path<'a>(
}
}
#[test]
fn package_should_be_built_false_for_empty_scripts() {
let dir = tempdir().unwrap();
let manifest = json!({ "name": "x", "version": "0.0.0" });
assert!(!package_should_be_built(&manifest, dir.path()));
}
#[test]
fn package_should_be_built_true_for_non_empty_prepare() {
let dir = tempdir().unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepare": "tsc" },
});
assert!(package_should_be_built(&manifest, dir.path()));
}
#[test]
fn package_should_be_built_false_when_main_exists_and_prepare_absent() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("index.js"), "").unwrap();
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepublish": "true" },
});
assert!(!package_should_be_built(&manifest, dir.path()));
}
#[test]
fn package_should_be_built_true_when_main_missing_and_prepublish_set() {
let dir = tempdir().unwrap();
// `main` defaults to `index.js`; create no file so it's missing.
let manifest = json!({
"name": "x", "version": "0.0.0",
"scripts": { "prepack": "rollup -c" },
});
assert!(package_should_be_built(&manifest, dir.path()));
}
#[test]
fn prepare_returns_should_be_built_false_when_no_manifest() {
let dir = tempdir().unwrap();
@@ -52,6 +52,7 @@ pub struct GitHostedTarballFetcher<'a> {
/// [`crate::GitFetcher`] for its other role.
pub package_id: &'a str,
pub requester: &'a str,
pub build_tools: crate::BuildTools<'a>,
}
impl GitHostedTarballFetcher<'_> {
@@ -107,6 +108,7 @@ impl GitHostedTarballFetcher<'_> {
self.store.files_index_file,
self.package_id,
self.scripts.ignore,
self.build_tools.identity(),
);
queue_files_index(
self.store.index_writer,
@@ -136,6 +138,7 @@ impl GitHostedTarballFetcher<'_> {
self.store.files_index_file,
self.package_id,
self.scripts.ignore,
self.build_tools.identity(),
);
queue_files_index(
self.store.index_writer,
@@ -158,6 +161,7 @@ impl<'a> GitHostedTarballFetcher<'a> {
extra_bin_paths: &[],
extra_env: &NO_EXTRA_ENV,
build_tools: self.build_tools,
}
}
}
@@ -47,6 +47,7 @@ async fn passes_through_package_without_scripts() {
);
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -102,6 +103,7 @@ async fn filters_files_outside_files_field() {
);
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -156,6 +158,7 @@ async fn rejects_build_when_not_allowed() {
);
let err = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -205,6 +208,7 @@ async fn surfaces_prepare_script_failure() {
);
let err = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -267,6 +271,7 @@ async fn path_field_packs_only_subdirectory() {
);
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -331,6 +336,7 @@ async fn materialized_temp_dir_does_not_corrupt_cas() {
let cas_bytes_before = fs::read(&original_cas_path).unwrap();
let _ = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -384,6 +390,7 @@ async fn writes_index_row_when_writer_provided() {
let key = "x@1.0.0\tbuilt";
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -478,6 +485,7 @@ async fn fast_path_returns_input_cas_paths_when_no_build_needed() {
let input_snapshot = cas_paths.clone();
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -537,6 +545,7 @@ async fn fast_path_queues_synthesized_index_row() {
let key = "x@1.0.0\tbuilt";
let _received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -626,6 +635,7 @@ async fn sub_path_never_takes_fast_path() {
let key = "sub@1.0.0\tbuilt";
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -713,6 +723,7 @@ async fn fast_path_skipped_preparation_caches_only_explicit_denials() {
let key = "x@1.0.0\tbuilt";
let received = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore,
unsafe_perm: true,
@@ -770,6 +781,7 @@ async fn tarball_path_traversal_attack_is_rejected() {
write_to_cas(&store_dir, &[("package.json", br#"{"name":"x","version":"1.0.0"}"#, false)]);
let err = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -827,6 +839,7 @@ async fn tarball_path_to_missing_subdir_is_rejected() {
write_to_cas(&store_dir, &[("package.json", br#"{"name":"x","version":"1.0.0"}"#, false)]);
let err = GitHostedTarballFetcher {
build_tools: crate::BuildTools::Detected,
scripts: crate::PrepareScriptOptions {
ignore: false,
unsafe_perm: true,
@@ -16,6 +16,7 @@ pnpm-diagnostics = { workspace = true }
pnpm-lockfile = { workspace = true }
pnpm-reporter = { workspace = true }
pnpm-resolving-resolver-base = { workspace = true }
pnpm-tool-protocol = { workspace = true }
chrono = { workspace = true }
derive_more = { workspace = true }
@@ -32,7 +32,7 @@ pub(super) fn collect_candidates(
let mut deduped: BTreeMap<String, Candidate> = BTreeMap::new();
let mut shape_violations = Vec::new();
for (key, metadata) in packages {
let name = key.name.clone();
let registry_package = registry_package_name(key, &metadata.resolution);
// A registry-qualified key (`<name>@<registryName>:<version>`)
// contributes its bare semver as the candidate version and the
// alias separately, so the verifier can route by registry while
@@ -49,9 +49,9 @@ pub(super) fn collect_candidates(
// that key shape, which is only sound while this invariant
// holds. The check is offline, so it applies even when no
// policy verifiers are active.
if has_registry_shape_mismatch(key, &metadata.resolution) {
if registry_package.is_none() || has_registry_shape_mismatch(key, &metadata.resolution) {
shape_violations.push(ResolutionPolicyViolation {
name: name.clone(),
name: key.name.clone(),
version: version.clone(),
resolution: metadata.resolution.clone(),
code: RESOLUTION_SHAPE_MISMATCH_VIOLATION_CODE,
@@ -59,6 +59,7 @@ pub(super) fn collect_candidates(
.to_string(),
});
}
let Some(name) = registry_package else { continue };
// Every `LockfileResolution` variant derives `Serialize`, and
// the wire shape never contains non-string keys or non-finite
// numbers — the only way this `expect` could fire is a future
@@ -83,6 +84,23 @@ pub(super) fn collect_candidates(
(deduped.into_values().collect(), shape_violations)
}
/// The registry package an entry is checked against: the key's own name,
/// except for a build tool (`yarn@tool:4.9.2`) published under another
/// package (`@yarnpkg/cli-dist`). `None` for a build tool entry whose
/// resolution is not the artifact that tool can be.
fn registry_package_name(
key: &pnpm_lockfile::PkgNameVerPeer,
resolution: &LockfileResolution,
) -> Option<PkgName> {
if key.suffix.prefix() != pnpm_lockfile::Prefix::Tool {
return Some(key.name.clone());
}
match pnpm_lockfile::build_tool_artifact(key, resolution)? {
pnpm_lockfile::BuildToolArtifact::Registry { package, .. } => package.parse().ok(),
pnpm_lockfile::BuildToolArtifact::Archive => Some(key.name.clone()),
}
}
/// A registry-style key must have a registry-shaped resolution because
/// build approval derives the trusted package identity from that key.
fn has_registry_shape_mismatch(
+129
View File
@@ -0,0 +1,129 @@
//! The tools a snapshot's `buildDependencies` record (`lockBuildTools`).
use crate::{
LockfileResolution, PackageKey, PackageMetadata, PkgName, Prefix, SnapshotEntry,
TarballResolution,
};
use pnpm_tool_protocol::{PackageManagerSource, package_manager_source};
use std::collections::HashMap;
/// The artifact a `tool:` entry is: the npm tarball of the package the tool
/// is published as, or the platform archives of a tool that ships them.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum BuildToolArtifact<'a> {
/// `package` is the npm package (`@yarnpkg/cli-dist` for Yarn 2 to 5).
Registry { package: &'static str, tarball: &'a TarballResolution },
/// Node.js, Bun, and Yarn 6 and later.
Archive,
}
/// The artifact the `tool:` entry `key` is, when `resolution` has the shape
/// the tool's source gives it. `None` for anything else: a key without the
/// `tool:` prefix, a tool pnpm does not provision, a tarball that is not an
/// integrity-pinned http(s) download, or a registry resolution, whose URL
/// would be derived from the tool's name rather than from its package.
#[must_use]
pub fn build_tool_artifact<'a>(
key: &PackageKey,
resolution: &'a LockfileResolution,
) -> Option<BuildToolArtifact<'a>> {
if key.suffix.prefix() != Prefix::Tool {
return None;
}
let name = key.name.to_string();
let source = if name == "node" {
PackageManagerSource::Archive
} else {
package_manager_source(&name, &key.suffix.version().to_string())?
};
match (source, resolution) {
(PackageManagerSource::Registry(package), LockfileResolution::Tarball(tarball))
if is_pinned_http_tarball(tarball) =>
{
Some(BuildToolArtifact::Registry { package, tarball })
}
(PackageManagerSource::Archive, archive) if is_downloaded_archive(archive) => {
Some(BuildToolArtifact::Archive)
}
_ => None,
}
}
fn is_pinned_http_tarball(tarball: &TarballResolution) -> bool {
tarball.integrity.is_some() && !tarball.is_git_hosted() && is_http_url(&tarball.tarball)
}
/// A binary archive, or platform archives, that are all downloads.
fn is_downloaded_archive(resolution: &LockfileResolution) -> bool {
match resolution {
LockfileResolution::Binary(binary) => is_http_url(&binary.url),
LockfileResolution::Variations(variations) => {
!variations.variants.is_empty()
&& variations.variants
.iter()
.all(|variant| is_downloaded_archive(&variant.resolution))
}
_ => false,
}
}
fn is_http_url(url: &str) -> bool {
let url = url.to_ascii_lowercase();
url.starts_with("https://") || url.starts_with("http://")
}
impl crate::Lockfile {
/// The `packages:` keys of the build tools that the snapshots at
/// `snapshot_keys` record. A walk that keeps those snapshots keeps
/// their tools' metadata too, though no install reaches the tools.
pub fn build_tool_keys<'a>(
&'a self,
snapshot_keys: impl IntoIterator<Item = &'a PackageKey> + 'a,
) -> impl Iterator<Item = PackageKey> + 'a {
snapshot_keys
.into_iter()
.filter_map(|key| self.snapshots.as_ref()?.get(key))
.flat_map(|snapshot| {
snapshot.build_dependency_keys().map(|(_, key)| key.without_peer())
})
}
}
impl SnapshotEntry {
/// The `packages:` / `snapshots:` key of every tool this snapshot
/// records, with the name it records the tool under.
pub fn build_dependency_keys(&self) -> impl Iterator<Item = (&PkgName, PackageKey)> {
self.build_dependencies
.iter()
.flatten()
.filter_map(|(alias, dep_ref)| Some((alias, dep_ref.resolve(alias)?)))
}
}
/// A digest naming every tool `snapshot` records by its key and resolution,
/// so two builds share it only when they ran on the same bytes. `None` when
/// the snapshot records no tools.
#[must_use]
pub fn build_tools_identity(
snapshot: &SnapshotEntry,
packages: Option<&HashMap<PackageKey, PackageMetadata>>,
) -> Option<String> {
let mut tools: Vec<String> = snapshot
.build_dependency_keys()
.map(|(alias, key)| {
let resolution = packages
.and_then(|packages| packages.get(&key))
.and_then(|metadata| serde_json::to_string(&metadata.resolution).ok())
.unwrap_or_default();
format!("{alias}={key}={resolution}")
})
.collect();
if tools.is_empty() {
return None;
}
tools.sort_unstable();
Some(pnpm_crypto_hash::create_hex_hash(&tools.join("\n")))
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,134 @@
use super::build_tools_identity;
use crate::{Lockfile, PackageKey, Prefix};
use pretty_assertions::assert_eq;
use text_block_macros::text_block;
const LOCKFILE: &str = text_block! {
"lockfileVersion: '9.0'"
""
"settings:"
" autoInstallPeers: true"
" excludeLinksFromLockfile: false"
" lockBuildTools: true"
""
"importers:"
""
" .:"
" dependencies:"
" example:"
" specifier: github:org/example#5c8f9a"
" version: git+https://github.com/org/example.git#5c8f9a"
""
"packages:"
""
" example@git+https://github.com/org/example.git#5c8f9a:"
" resolution: {commit: 5c8f9a, repo: https://github.com/org/example.git, type: git}"
" version: 1.0.0"
""
" yarn@tool:1.22.22:"
" resolution: {integrity: sha512-mQ==, tarball: https://registry.npmjs.org/yarn/-/yarn-1.22.22.tgz}"
""
"snapshots:"
""
" example@git+https://github.com/org/example.git#5c8f9a:"
" buildDependencies:"
" yarn: tool:1.22.22"
""
};
fn parse(text: &str) -> Lockfile {
serde_saphyr::from_str(text).expect("parse lockfile")
}
fn example_key() -> PackageKey {
"example@git+https://github.com/org/example.git#5c8f9a".parse().expect("parse key")
}
#[test]
fn build_dependencies_round_trip() {
let lockfile = parse(LOCKFILE);
let saved = lockfile.to_yaml_string().expect("serialize lockfile");
assert_eq!(saved, LOCKFILE);
}
#[test]
fn build_dependency_keys_carry_the_tool_prefix() {
let lockfile = parse(LOCKFILE);
let snapshot = &lockfile.snapshots.as_ref().expect("snapshots")[&example_key()];
let keys: Vec<_> = snapshot.build_dependency_keys().collect();
dbg!(&keys);
assert_eq!(keys.len(), 1);
let (alias, key) = &keys[0];
assert_eq!(alias.to_string(), "yarn");
assert_eq!(key.to_string(), "yarn@tool:1.22.22");
assert_eq!(key.suffix.prefix(), Prefix::Tool);
}
#[test]
fn build_tools_identity_follows_the_recorded_bytes() {
let lockfile = parse(LOCKFILE);
let tampered = parse(&LOCKFILE.replace("sha512-mQ==", "sha512-mg=="));
let identity = |lockfile: &Lockfile| {
let snapshot = &lockfile.snapshots.as_ref().expect("snapshots")[&example_key()];
build_tools_identity(snapshot, lockfile.packages.as_ref())
};
let recorded = identity(&lockfile);
assert!(recorded.is_some());
assert_eq!(identity(&lockfile), recorded);
assert_ne!(identity(&tampered), recorded);
assert_eq!(
build_tools_identity(&crate::SnapshotEntry::default(), lockfile.packages.as_ref()),
None,
);
}
/// What a frozen install will run as a build tool must be the tool's own
/// artifact: anything else in the lockfile is refused.
#[test]
fn a_build_tool_entry_must_be_the_tools_own_artifact() {
let artifact = |key: &str, resolution: &str| {
let resolution: crate::LockfileResolution =
serde_saphyr::from_str(resolution).expect("parse resolution");
let key: PackageKey = key.parse().expect("parse key");
super::build_tool_artifact(&key, &resolution)
.map(|artifact| match artifact {
super::BuildToolArtifact::Registry { package, .. } => package,
super::BuildToolArtifact::Archive => "archive",
})
};
let berry = "{integrity: sha512-mQ==, tarball: https://registry.npmjs.org/@yarnpkg/cli-dist/-/cli-dist-4.9.2.tgz}";
let variations = text_block! {
"type: variations"
"variants:"
" - resolution: {type: binary, archive: tarball, bin: bin/node, integrity: sha512-mQ==, url: https://nodejs.org/dist/v24.6.0/node-v24.6.0-linux-x64.tar.gz}"
" targets: [{os: linux, cpu: x64}]"
};
let local_variations = variations.replace("https://nodejs.org/dist/v24.6.0", "file:.");
assert_eq!(artifact("yarn@tool:4.9.2", berry), Some("@yarnpkg/cli-dist"));
assert_eq!(artifact("yarn@tool:1.22.22", berry), Some("yarn"));
assert_eq!(artifact("yarn@tool:6.0.0", variations), Some("archive"));
assert_eq!(artifact("node@tool:24.6.0", variations), Some("archive"));
assert_eq!(artifact("node@tool:24.6.0", &local_variations), None);
assert_eq!(artifact("node@tool:24.6.0", "{type: variations, variants: []}"), None);
// A local file, a tarball without integrity, and a registry resolution
// whose URL would come from the tool's name.
assert_eq!(
artifact("yarn@tool:1.22.22", "{tarball: 'file:./yarn.tgz', integrity: sha512-mQ==}"),
None,
);
assert_eq!(
artifact(
"yarn@tool:1.22.22",
"{tarball: https://registry.npmjs.org/yarn/-/yarn-1.22.22.tgz}"
),
None,
);
assert_eq!(artifact("yarn@tool:4.9.2", "{integrity: sha512-mQ==}"), None);
// A tool in the wrong shape, an unknown tool, and a key without `tool:`.
assert_eq!(artifact("node@tool:24.6.0", berry), None);
assert_eq!(artifact("yarn@tool:6.0.0", berry), None);
assert_eq!(artifact("evil@tool:1.0.0", berry), None);
assert_eq!(artifact("yarn@1.22.22", berry), None);
}
@@ -110,10 +110,11 @@ impl Lockfile {
}
let reachable = collect_reachable(&filtered, seeds, options, &peer_edges)?;
let reachable_metadata: HashSet<_> = reachable
let mut reachable_metadata: HashSet<_> = reachable
.iter()
.map(PkgNameVerPeer::without_peer)
.collect();
reachable_metadata.extend(filtered.build_tool_keys(&reachable));
if let Some(snapshots) = filtered.snapshots.as_mut() {
snapshots.retain(|key, _| reachable.contains(key));
peer_edges.prune_dangling(snapshots);
+12 -1
View File
@@ -18,7 +18,7 @@ pub use manifest::satisfies_package_manifest;
pub use recorded_settings::{
auto_install_peers_changed, exclude_links_from_lockfile_changed,
recorded_add_missing_peer_types, recorded_dedupe_peers, recorded_inject_workspace_packages,
recorded_peers_suffix_max_length,
recorded_lock_build_tools, recorded_peers_suffix_max_length,
};
pub use spec_diff::SpecDiff;
@@ -36,6 +36,7 @@ pub struct LockfileSettingsCheck<'a> {
pub package_extensions_checksum: Option<&'a str>,
pub ignored_optional_dependencies: Option<&'a [String]>,
pub patched_dependencies: Option<&'a BTreeMap<String, String>>,
pub lock_build_tools: bool,
pub resolution: ResolutionSettingsCheck<'a>,
}
@@ -259,6 +260,14 @@ pub enum StalenessReason {
)]
AddMissingPeerTypesChanged { lockfile: bool, config: bool },
/// The lockfile's `settings.lockBuildTools` differs from the current
/// install's `Config::lock_build_tools`. Normalized like
/// [`Self::DedupePeersChanged`].
#[display(
"`lockBuildTools` in the lockfile ({lockfile}) doesn't match the current config ({config})"
)]
LockBuildToolsChanged { lockfile: bool, config: bool },
/// The lockfile's `settings.excludeLinksFromLockfile` differs from
/// the current install's `Config::exclude_links_from_lockfile`. The
/// setting decides whether `link:` deps are recorded at all, so a
@@ -321,6 +330,7 @@ impl StalenessReason {
StalenessReason::AutoInstallPeersChanged { .. }
| StalenessReason::DedupePeersChanged { .. }
| StalenessReason::AddMissingPeerTypesChanged { .. }
| StalenessReason::LockBuildToolsChanged { .. }
| StalenessReason::ExcludeLinksFromLockfileChanged { .. }
| StalenessReason::PeersSuffixMaxLengthChanged { .. }
| StalenessReason::InjectWorkspacePackagesChanged { .. } => self.settings_block_key(),
@@ -368,6 +378,7 @@ pub fn check_lockfile_settings(
) -> Result<(), StalenessReason> {
check_recorded_config(lockfile, &check)?;
check_recorded_settings(lockfile, &check.resolution)?;
recorded_settings::check_lock_build_tools(lockfile, check.lock_build_tools)?;
// Last, because every gate above reports a cause this one cannot: each
// compares the lockfile against a configured value the user can act on,
// while this compares the lockfile against itself. A lockfile that drifted
@@ -14,6 +14,7 @@ impl StalenessReason {
StalenessReason::AddMissingPeerTypesChanged { .. } => {
Some("settings.addMissingPeerTypes")
}
StalenessReason::LockBuildToolsChanged { .. } => Some("settings.lockBuildTools"),
StalenessReason::ExcludeLinksFromLockfileChanged { .. } => {
Some("settings.excludeLinksFromLockfile")
}
@@ -69,6 +70,21 @@ pub(super) fn check_recorded_settings(
check_resolution_settings(settings, check.resolution_settings)
}
/// `settings.lockBuildTools` against the current configuration.
pub(super) fn check_lock_build_tools(
lockfile: &Lockfile,
configured: bool,
) -> Result<(), StalenessReason> {
let recorded = recorded_lock_build_tools(lockfile.settings.as_ref());
if recorded != configured {
return Err(StalenessReason::LockBuildToolsChanged {
lockfile: recorded,
config: configured,
});
}
Ok(())
}
fn check_peer_settings(
settings: Option<&LockfileSettings>,
check: &ResolutionSettingsCheck<'_>,
@@ -175,6 +191,12 @@ pub fn recorded_add_missing_peer_types(recorded: Option<&LockfileSettings>) -> b
recorded.and_then(|settings| settings.add_missing_peer_types).unwrap_or(false)
}
/// See [`auto_install_peers_changed`].
#[must_use]
pub fn recorded_lock_build_tools(recorded: Option<&LockfileSettings>) -> bool {
recorded.and_then(|settings| settings.lock_build_tools).unwrap_or(false)
}
/// See [`auto_install_peers_changed`].
#[must_use]
pub fn recorded_peers_suffix_max_length(recorded: Option<&LockfileSettings>) -> u64 {
@@ -29,6 +29,7 @@ fn settings_check(catalogs: &Catalogs) -> LockfileSettingsCheck<'_> {
package_extensions_checksum: None,
ignored_optional_dependencies: None,
patched_dependencies: None,
lock_build_tools: false,
resolution: crate::freshness::ResolutionSettingsCheck {
auto_install_peers: true,
dedupe_peers: false,
+8
View File
@@ -1,6 +1,7 @@
#![cfg_attr(dylint_lib = "perfectionist", feature(register_tool))]
#![cfg_attr(dylint_lib = "perfectionist", register_tool(perfectionist))]
pub use build_tools::*;
pub use catalog_snapshots::*;
pub use comver::*;
pub use env_lockfile::*;
@@ -30,6 +31,7 @@ pub use snapshot_dep_ref::*;
pub use snapshot_entry::*;
pub use yaml_documents::*;
mod build_tools;
mod catalog_snapshots;
mod comver;
mod env_lockfile;
@@ -93,6 +95,12 @@ pub struct LockfileSettings {
/// [`Self::dedupe_peers`].
#[serde(default, skip_serializing_if = "Option::is_none")]
pub add_missing_peer_types: Option<bool>,
/// Recorded as `Some(true)` when the install ran with `lockBuildTools`
/// on, omitted otherwise, like [`Self::dedupe_peers`]. It says that
/// every git-hosted package that needs a build carries its
/// [`buildDependencies`](crate::SnapshotEntry::build_dependencies).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lock_build_tools: Option<bool>,
/// `injectWorkspacePackages` recorded by the install that wrote
/// this lockfile. `false` round-trips as a missing key — the key
/// is stripped on save so historic v9 lockfiles (which never
@@ -20,6 +20,7 @@ pub(super) fn merge_settings(
exclude_links_from_lockfile: ours.exclude_links_from_lockfile
|| theirs.exclude_links_from_lockfile,
add_missing_peer_types: ours.add_missing_peer_types.or(theirs.add_missing_peer_types),
lock_build_tools: ours.lock_build_tools.or(theirs.lock_build_tools),
inject_workspace_packages: ours.inject_workspace_packages
|| theirs.inject_workspace_packages,
peers_suffix_max_length: ours.peers_suffix_max_length
+18 -11
View File
@@ -70,9 +70,8 @@ pub struct PkgVerPeer {
/// consumers can discriminate runtime entries by their prefix instead of
/// substring-searching the depPath.
///
/// The enum is closed because pnpm only defines this one scheme so
/// far; if upstream adds another (e.g. `tag:`), add a variant here
/// rather than turning the prefix into an unbounded string.
/// The enum is closed: add a variant here for a new scheme rather than
/// turning the prefix into an unbounded string.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Prefix {
/// No prefix — plain semver.
@@ -80,6 +79,10 @@ pub enum Prefix {
/// `runtime:` — runtime dependency specifier (`node@runtime:`,
/// `deno@runtime:`, `bun@runtime:`).
Runtime,
/// `tool:` — a tool pnpm provisions to build a package
/// (`node@tool:24.6.0`, `yarn@tool:4.9.2`), reached only through a
/// snapshot's [`buildDependencies`](crate::SnapshotEntry::build_dependencies).
Tool,
}
impl Prefix {
@@ -89,7 +92,8 @@ impl Prefix {
pub const fn as_str(self) -> &'static str {
match self {
Prefix::None => "",
Prefix::Runtime => "runtime:",
Prefix::Runtime => pnpm_tool_protocol::RUNTIME_PROTOCOL,
Prefix::Tool => pnpm_tool_protocol::TOOL_PROTOCOL,
}
}
}
@@ -186,7 +190,7 @@ pub enum ParsePkgVerPeerError {
MismatchParenthesis,
#[display("Empty path after `file:` scheme")]
EmptyFilePath,
#[display("`runtime:` and `file:` schemes are mutually exclusive")]
#[display("`runtime:` / `tool:` and `file:` schemes are mutually exclusive")]
ConflictingSchemes,
}
@@ -224,16 +228,19 @@ impl FromStr for PkgVerPeer {
// scheme prefix flush against the semver, e.g.
// `runtime:22.0.0` — splitting it off here keeps the
// parenthesis-based peer-suffix detection below unchanged.
let (prefix, body) = if let Some(rest) = value.strip_prefix(Prefix::Runtime.as_str()) {
(Prefix::Runtime, rest)
} else {
(Prefix::None, value)
};
let (prefix, body) = [Prefix::Runtime, Prefix::Tool]
.into_iter()
.find_map(|prefix| {
value
.strip_prefix(prefix.as_str())
.map(|rest| (prefix, rest))
})
.unwrap_or((Prefix::None, value));
// Pnpm never writes `runtime:file:...`; rejecting it here keeps
// [`PkgVerPeer::to_string`] byte-stable (a silent acceptance would
// round-trip back as `file:...` with the prefix dropped).
if prefix == Prefix::Runtime && body.starts_with("file:") {
if prefix != Prefix::None && body.starts_with("file:") {
return Err(ParsePkgVerPeerError::ConflictingSchemes);
}
+14 -1
View File
@@ -66,7 +66,8 @@ fn parse_err() {
case!("1.21.3(" => "Mismatch parenthesis", ParsePkgVerPeerError::MismatchParenthesis);
case!("1.21.3)" => "Mismatch parenthesis", ParsePkgVerPeerError::MismatchParenthesis);
case!("file:" => "Empty path after `file:` scheme", ParsePkgVerPeerError::EmptyFilePath);
case!("runtime:file:packages/pkg" => "`runtime:` and `file:` schemes are mutually exclusive", ParsePkgVerPeerError::ConflictingSchemes);
case!("runtime:file:packages/pkg" => "`runtime:` / `tool:` and `file:` schemes are mutually exclusive", ParsePkgVerPeerError::ConflictingSchemes);
case!("tool:file:packages/pkg" => "`runtime:` / `tool:` and `file:` schemes are mutually exclusive", ParsePkgVerPeerError::ConflictingSchemes);
}
#[test]
@@ -157,6 +158,18 @@ fn parse_runtime_prefix_with_peer_suffix() {
assert_eq!(parsed.to_string(), "runtime:22.0.0(node@22.0.0)");
}
/// A build tool's key (`yarn@tool:4.9.2`) parses as a `tool:` version,
/// not as a version from a registry named `tool`.
#[test]
fn parse_tool_prefix_round_trips() {
let parsed: PkgVerPeer = "tool:4.9.2".parse().expect("parse tool version");
dbg!(&parsed);
assert_eq!(parsed.prefix(), Prefix::Tool);
assert_eq!(parsed.version_semver(), Some(&"4.9.2".parse::<Version>().unwrap()));
assert_eq!(parsed.registry_qualified(), None);
assert_eq!(parsed.to_string(), "tool:4.9.2");
}
#[test]
fn parse_bare_semver_has_no_prefix() {
let parsed: PkgVerPeer = "1.21.3".parse().expect("parse bare");
+11
View File
@@ -380,6 +380,17 @@ pub enum LockfileResolution {
}
impl LockfileResolution {
/// Whether the package comes from a git repository, either cloned or
/// downloaded as a git host's archive, and so is prepared on install.
#[must_use]
pub fn is_git_hosted(&self) -> bool {
match self {
LockfileResolution::Git(_) => true,
LockfileResolution::Tarball(tarball) => tarball.is_git_hosted(),
_ => false,
}
}
/// Get the integrity field if available.
#[must_use]
pub fn integrity(&self) -> Option<&'_ Integrity> {
@@ -263,6 +263,7 @@ fn peers_suffix_max_length_omitted_from_settings_when_unset() {
lockfile_version: crate::LockfileVersion::<9>::try_from(crate::ComVer::new(9, 0))
.expect("v9 is compatible with major=9"),
settings: Some(LockfileSettings {
lock_build_tools: None,
auto_install_peers: true,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -303,6 +304,7 @@ fn peers_suffix_max_length_serialized_when_set() {
lockfile_version: crate::LockfileVersion::<9>::try_from(crate::ComVer::new(9, 0))
.expect("v9 is compatible with major=9"),
settings: Some(LockfileSettings {
lock_build_tools: None,
auto_install_peers: true,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -26,6 +26,17 @@ pub struct SnapshotEntry {
serialize_with = "crate::serialize_yaml::sorted_map_opt"
)]
pub optional_dependencies: Option<HashMap<PkgName, SnapshotDepRef>>,
/// The tools pnpm provisions to build this package, such as the
/// package manager and Node.js that prepare a git-hosted dependency.
///
/// No install follows these edges: a build tool is never linked into
/// `node_modules`. Pruning the lockfile does follow them, so a tool
/// stays recorded for as long as a package references it.
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "crate::serialize_yaml::sorted_map_opt"
)]
pub build_dependencies: Option<HashMap<PkgName, SnapshotDepRef>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub transitive_peer_dependencies: Option<Vec<String>>,
+2 -1
View File
@@ -46,7 +46,7 @@ const INDENT: usize = 2;
/// Per-package / per-snapshot key priority, matching pnpm's lockfile
/// key sort.
const ORDERED_KEYS: [&str; 20] = [
const ORDERED_KEYS: [&str; 21] = [
"resolution",
"id",
"name",
@@ -64,6 +64,7 @@ const ORDERED_KEYS: [&str; 20] = [
"peerDependenciesMeta",
"dependencies",
"optionalDependencies",
"buildDependencies",
"transitivePeerDependencies",
"dev",
"optional",
@@ -102,6 +102,7 @@ pub fn build_package_snapshot(
};
let snapshot = SnapshotEntry {
build_dependencies: None,
id: None,
dependencies: (!dependencies.is_empty()).then_some(dependencies),
optional_dependencies: None,
@@ -0,0 +1,219 @@
//! Record in the lockfile the tools that prepare each git-hosted package
//! (`lockBuildTools`).
//!
//! A git-hosted snapshot the previous lockfile already recorded keeps its
//! record, so the tools change only when the package re-resolves. Every
//! other one is read at its locked commit: when it needs a build, the
//! package manager and the Node.js it asks for are resolved and written as
//! `tool:` entries, reached through the snapshot's `buildDependencies`.
pub use errors::RecordBuildToolsError;
pub(crate) use package_read::PackageSources;
mod errors;
mod package_read;
mod tool_resolution;
use pnpm_lockfile::{
Lockfile, PackageKey, PackageMetadata, PkgName, SnapshotDepRef, SnapshotEntry,
};
use pnpm_resolving_resolver_base::{ResolveOptions, Resolver};
use std::collections::HashMap;
/// The tools one package records: each one's name in `buildDependencies`,
/// its key, and its `packages:` entry.
type BuildToolRecord = HashMap<PkgName, (PackageKey, PackageMetadata)>;
/// The tool entries known so far: the previous lockfile's, and those this
/// pass resolved. Packages that need the same tool share one entry.
type ToolCatalog = HashMap<PackageKey, PackageMetadata>;
/// What records the tools: the resolver chain of the install, kept past
/// resolution only while this runs, and where the packages are read from.
pub(crate) struct BuildToolRecorder<'a> {
pub resolver: &'a dyn Resolver,
pub resolve_options: &'a ResolveOptions,
pub sources: PackageSources<'a>,
}
impl BuildToolRecorder<'_> {
/// Give every git-hosted snapshot of `lockfile` that needs a build its
/// `buildDependencies`, carrying records over from `prior`.
pub(crate) async fn record<Reporter: pnpm_reporter::Reporter>(
&self,
lockfile: &mut Lockfile,
prior: Option<&Lockfile>,
) -> Result<(), RecordBuildToolsError> {
let prior = prior
.filter(|prior| pnpm_lockfile::recorded_lock_build_tools(prior.settings.as_ref()))
.map(PriorRecords::of);
let mut records: HashMap<PackageKey, Option<BuildToolRecord>> = HashMap::new();
let mut unrecorded = Vec::new();
for metadata_key in git_hosted_metadata_keys(lockfile) {
match prior
.as_ref()
.and_then(|prior| prior.carried(lockfile, &metadata_key))
{
Some(record) => drop(records.insert(metadata_key, record)),
None => unrecorded.push(metadata_key),
}
}
let lockfile_view = &*lockfile;
let environments = futures_util::future::try_join_all(
unrecorded
.iter()
.map(|key| self.read_environment::<Reporter>(lockfile_view, key)),
)
.await?;
let mut catalog = prior
.as_ref()
.map(PriorRecords::tools)
.unwrap_or_default();
for (metadata_key, environment) in unrecorded.into_iter().zip(environments) {
let record = match environment {
Some(environment) => Some(self.resolve_tools(&mut catalog, &environment).await?),
None => None,
};
records.insert(metadata_key, record);
}
apply_records(lockfile, &records);
Ok(())
}
async fn read_environment<Reporter: pnpm_reporter::Reporter>(
&self,
lockfile: &Lockfile,
metadata_key: &PackageKey,
) -> Result<Option<pnpm_git_fetcher::BuildEnvironment>, RecordBuildToolsError> {
let Some(metadata) = lockfile.packages
.as_ref()
.and_then(|packages| packages.get(metadata_key))
else {
return Ok(None);
};
self.sources.read_build_environment::<Reporter>(metadata_key, metadata).await
}
}
/// The `packages:` keys of the git-hosted packages that have snapshots.
fn git_hosted_metadata_keys(lockfile: &Lockfile) -> Vec<PackageKey> {
let (Some(packages), Some(snapshots)) = (&lockfile.packages, &lockfile.snapshots) else {
return Vec::new();
};
let mut keys: Vec<PackageKey> = snapshots
.keys()
.map(PackageKey::without_peer)
.filter(|key| {
packages.get(key).is_some_and(|metadata| metadata.resolution.is_git_hosted())
})
.collect();
keys.sort_by_key(ToString::to_string);
keys.dedup();
keys
}
/// The previous lockfile, with a snapshot of each package at hand.
struct PriorRecords<'a> {
lockfile: &'a Lockfile,
snapshots_by_package: HashMap<PackageKey, &'a SnapshotEntry>,
}
impl<'a> PriorRecords<'a> {
fn of(lockfile: &'a Lockfile) -> Self {
let snapshots_by_package = lockfile.snapshots
.iter()
.flatten()
.map(|(key, snapshot)| (key.without_peer(), snapshot))
.collect();
PriorRecords { lockfile, snapshots_by_package }
}
/// The tools the previous lockfile's git-hosted snapshots record, which
/// a package that needs one of them reuses when its version satisfies
/// the package. An entry no such snapshot names is not offered.
fn tools(&self) -> ToolCatalog {
let Some(packages) = self.lockfile.packages.as_ref() else {
return ToolCatalog::new();
};
self.snapshots_by_package
.iter()
.filter(|(package, _)| {
packages
.get(*package)
.is_some_and(|metadata| metadata.resolution.is_git_hosted())
})
.flat_map(|(_, snapshot)| {
snapshot.build_dependency_keys().map(|(_, key)| key.without_peer())
})
.filter_map(|key| {
let metadata = packages.get(&key)?;
pnpm_lockfile::build_tool_artifact(&key, &metadata.resolution)?;
Some((key, metadata.clone()))
})
.collect()
}
/// What the previous lockfile recorded for the package at
/// `metadata_key`, when it holds the same package and every tool it
/// names is that tool's own artifact. `Some(None)` is a package it
/// found needs no build.
fn carried(
&self,
lockfile: &Lockfile,
metadata_key: &PackageKey,
) -> Option<Option<BuildToolRecord>> {
let resolution = |lockfile: &Lockfile| {
lockfile.packages
.as_ref()?
.get(metadata_key)
.map(|metadata| metadata.resolution.clone())
};
if resolution(self.lockfile)? != resolution(lockfile)? {
return None;
}
let snapshot = self.snapshots_by_package.get(metadata_key)?;
if snapshot.build_dependencies.is_none() {
return Some(None);
}
let packages = self.lockfile.packages.as_ref()?;
let record = snapshot
.build_dependency_keys()
.map(|(alias, key)| {
let metadata = packages
.get(&key.without_peer())?
.clone();
pnpm_lockfile::build_tool_artifact(&key, &metadata.resolution)?;
Some((alias.clone(), (key, metadata)))
})
.collect::<Option<BuildToolRecord>>()?;
Some(Some(record))
}
}
/// Write `records` into `lockfile`: each package's snapshots get its
/// `buildDependencies`, and each tool its `packages:` entry.
fn apply_records(lockfile: &mut Lockfile, records: &HashMap<PackageKey, Option<BuildToolRecord>>) {
let mut tools: HashMap<PackageKey, PackageMetadata> = HashMap::new();
for (key, snapshot) in lockfile.snapshots.iter_mut().flatten() {
let Some(record) = records.get(&key.without_peer()) else { continue };
snapshot.build_dependencies = record
.as_ref()
.map(|record| {
record
.iter()
.map(|(alias, (tool_key, _))| {
(alias.clone(), SnapshotDepRef::Plain(tool_key.suffix.clone()))
})
.collect()
});
for (tool_key, metadata) in record.iter().flat_map(HashMap::values) {
tools.insert(tool_key.clone(), metadata.clone());
}
}
// A tool gets no `snapshots:` entry: an install materializes every
// snapshot it holds, and a tool is never installed.
lockfile.packages.get_or_insert_default().extend(tools);
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,38 @@
use derive_more::{Display, Error};
use miette::Diagnostic;
/// Why the build tools of git-hosted packages could not be recorded.
#[derive(Debug, Display, Error, Diagnostic)]
pub enum RecordBuildToolsError {
#[diagnostic(transparent)]
ReadGitPackage(#[error(source)] pnpm_git_fetcher::GitFetcherError),
#[diagnostic(transparent)]
ReadArchive(#[error(source)] pnpm_tarball::TarballError),
#[display("Failed to read the manifest of {package} from the store: {source}")]
#[diagnostic(code(ERR_PNPM_BUILD_TOOLS_READ_MANIFEST))]
ReadArchiveFile {
package: String,
#[error(source)]
source: std::io::Error,
},
#[display("Failed to read the manifest of {package}: {source}")]
#[diagnostic(code(ERR_PNPM_BUILD_TOOLS_READ_MANIFEST))]
ReadManifest {
package: String,
#[error(source)]
source: serde_json::Error,
},
#[display("Failed to resolve {tool}, which builds a git-hosted dependency: {reason}")]
#[diagnostic(code(ERR_PNPM_BUILD_TOOL_RESOLUTION))]
Resolve { tool: String, reason: String },
#[display(
"No version of {tool} matches, so the git-hosted dependency that asks for it cannot be built"
)]
#[diagnostic(code(ERR_PNPM_BUILD_TOOL_RESOLUTION))]
NoMatch { tool: String },
}
@@ -0,0 +1,128 @@
//! Read a git-hosted package at its locked commit to learn what its build
//! needs.
use super::RecordBuildToolsError;
use pnpm_config::Config;
use pnpm_git_fetcher::{
BuildEnvironment, CasPackageFiles, GitManifestQuery, GitSourceCache, detect_build_environment,
read_git_build_environment,
};
use pnpm_lockfile::{LockfileResolution, PackageKey, PackageMetadata, TarballResolution};
use pnpm_network::ThrottledClient;
use pnpm_reporter::Reporter;
use pnpm_store_dir::SharedVerifiedFilesCache;
use pnpm_tarball::{IngestTarballToStore, MemCache};
use std::{collections::HashMap, path::PathBuf, sync::Arc};
/// Where a git-hosted package's files come from: the checkout the install
/// shares between resolution and fetch, or the archive a git host serves,
/// read through the install's tarball cache so it is downloaded once.
#[derive(Clone, Copy)]
pub(crate) struct PackageSources<'a> {
pub config: &'static Config,
pub http_client: &'a ThrottledClient,
pub git_source_cache: &'a GitSourceCache,
pub tarball_mem_cache: &'a MemCache,
pub store_index: Option<&'a pnpm_store_dir::SharedReadonlyStoreIndex>,
}
impl PackageSources<'_> {
/// What the package at `metadata_key` needs to be built. `None` when
/// it needs no build.
pub(super) async fn read_build_environment<Reporter: self::Reporter>(
&self,
metadata_key: &PackageKey,
metadata: &PackageMetadata,
) -> Result<Option<BuildEnvironment>, RecordBuildToolsError> {
match &metadata.resolution {
LockfileResolution::Git(git) => read_git_build_environment(GitManifestQuery {
source_cache: self.git_source_cache,
repo: &git.repo,
commit: &git.commit,
path: git.path.as_deref(),
git_shallow_hosts: &self.config.git_shallow_hosts,
git_bin: None,
git_config: &[],
})
.await
.map_err(RecordBuildToolsError::ReadGitPackage),
LockfileResolution::Tarball(tarball) => {
let files = self.archive_files::<Reporter>(tarball).await?;
archive_build_environment(metadata_key, &files, tarball.path.as_deref())
}
_ => Ok(None),
}
}
async fn archive_files<Reporter: self::Reporter>(
&self,
tarball: &TarballResolution,
) -> Result<Arc<HashMap<String, PathBuf>>, RecordBuildToolsError> {
let config = self.config;
IngestTarballToStore {
fetching: pnpm_tarball::ArchiveFetchOptions {
http_client: self.http_client,
auth_headers: &config.auth_headers,
retry_opts: config.retry_opts(),
offline: config.offline,
},
package: pnpm_tarball::TarballPackage {
integrity: tarball.integrity.as_ref(),
unpacked_size: None,
file_count: None,
url: &tarball.tarball,
// The id resolution read the archive under, so the store
// index row it left answers this read too.
id: &tarball.tarball,
},
store: pnpm_tarball::ArchiveStoreContext {
dir: &config.store_dir,
index: self.store_index.cloned(),
index_writer: None,
verify_integrity: config.verify_store_integrity,
strict_pkg_content_check: config.strict_store_pkg_content_check,
verified_files_cache: SharedVerifiedFilesCache::default(),
prefetched_cas_paths: None,
},
requester: "",
ignore_file_pattern: None,
progress_reported: None,
store_projection: pnpm_tarball::ArchiveStoreProjection::Package {
append_manifest: None,
},
}
.run_with_mem_cache::<Reporter>(self.tarball_mem_cache)
.await
.map_err(RecordBuildToolsError::ReadArchive)
}
}
/// The build environment of the package in the `dir` of a git host's
/// archive.
fn archive_build_environment(
metadata_key: &PackageKey,
files: &HashMap<String, PathBuf>,
dir: Option<&str>,
) -> Result<Option<BuildEnvironment>, RecordBuildToolsError> {
let repository = CasPackageFiles { files, dir: None };
let package = CasPackageFiles { files, dir };
let Some(manifest_path) = files.get(&archive_manifest_path(dir)) else { return Ok(None) };
let manifest = std::fs::read(manifest_path)
.map_err(|source| RecordBuildToolsError::ReadArchiveFile {
package: metadata_key.to_string(),
source,
})?;
let manifest = pnpm_package_manifest::parse_manifest_bytes(&manifest)
.map_err(|source| RecordBuildToolsError::ReadManifest {
package: metadata_key.to_string(),
source,
})?;
Ok(detect_build_environment(&repository, &package, &manifest))
}
fn archive_manifest_path(dir: Option<&str>) -> String {
match dir.unwrap_or("").trim_matches('/') {
"" => "package.json".to_string(),
dir => format!("{dir}/package.json"),
}
}
@@ -0,0 +1,196 @@
use super::{PriorRecords, apply_records, git_hosted_metadata_keys};
use pnpm_lockfile::{Lockfile, PackageKey};
use pretty_assertions::assert_eq;
use std::collections::HashMap;
use text_block_macros::text_block;
const RECORDED: &str = text_block! {
"lockfileVersion: '9.0'"
"settings:"
" autoInstallPeers: true"
" excludeLinksFromLockfile: false"
" lockBuildTools: true"
"importers:"
" .:"
" dependencies:"
" built:"
" specifier: git+file:///repos/built#1111111111111111111111111111111111111111"
" version: git+file:///repos/built#1111111111111111111111111111111111111111"
" plain:"
" specifier: git+file:///repos/plain#2222222222222222222222222222222222222222"
" version: git+file:///repos/plain#2222222222222222222222222222222222222222"
" is-positive:"
" specifier: 1.0.0"
" version: 1.0.0"
"packages:"
" built@git+file:///repos/built#1111111111111111111111111111111111111111:"
" resolution: {commit: '1111111111111111111111111111111111111111', repo: file:///repos/built, type: git}"
" version: 1.0.0"
" plain@git+file:///repos/plain#2222222222222222222222222222222222222222:"
" resolution: {commit: '2222222222222222222222222222222222222222', repo: file:///repos/plain, type: git}"
" version: 1.0.0"
" is-positive@1.0.0:"
" resolution: {integrity: sha512-mQ==}"
" yarn@tool:1.22.22:"
" resolution: {integrity: sha512-yQ==, tarball: https://registry.npmjs.org/yarn/-/yarn-1.22.22.tgz}"
"snapshots:"
" built@git+file:///repos/built#1111111111111111111111111111111111111111:"
" buildDependencies:"
" yarn: tool:1.22.22"
" plain@git+file:///repos/plain#2222222222222222222222222222222222222222: {}"
" is-positive@1.0.0: {}"
};
fn parse(text: &str) -> Lockfile {
serde_saphyr::from_str(text).expect("parse lockfile")
}
fn key(text: &str) -> PackageKey {
text.parse().expect("parse key")
}
const BUILT: &str = "built@git+file:///repos/built#1111111111111111111111111111111111111111";
const PLAIN: &str = "plain@git+file:///repos/plain#2222222222222222222222222222222222222222";
/// A lockfile as the resolver writes it: the records dropped.
fn unrecorded() -> Lockfile {
let mut lockfile = parse(RECORDED);
for snapshot in lockfile.snapshots
.iter_mut()
.flatten()
.map(|(_, snapshot)| snapshot)
{
snapshot.build_dependencies = None;
}
let tool = key("yarn@tool:1.22.22");
lockfile.packages
.as_mut()
.unwrap()
.remove(&tool);
lockfile.snapshots
.as_mut()
.unwrap()
.remove(&tool);
lockfile
}
#[test]
fn only_git_hosted_packages_are_examined() {
let keys: Vec<String> = git_hosted_metadata_keys(&unrecorded())
.iter()
.map(ToString::to_string)
.collect();
assert_eq!(keys, [BUILT, PLAIN]);
}
/// A package the previous lockfile already examined keeps what it found,
/// a record or the absence of one.
#[test]
fn records_carry_over_for_an_unchanged_package() {
let prior = parse(RECORDED);
let lockfile = unrecorded();
let built = PriorRecords::of(&prior)
.carried(&lockfile, &key(BUILT))
.expect("carried");
let built = built.expect("a record");
let tools: Vec<String> = built
.values()
.map(|(tool_key, _)| tool_key.to_string())
.collect();
assert_eq!(tools, ["yarn@tool:1.22.22"]);
assert_eq!(
PriorRecords::of(&prior)
.carried(&lockfile, &key(PLAIN))
.map(|record| record.is_none()),
Some(true),
);
}
#[test]
fn a_package_the_previous_lockfile_does_not_hold_is_examined_again() {
let prior = parse(RECORDED);
let mut lockfile = unrecorded();
let packages = lockfile.packages.as_mut().unwrap();
let mut metadata = packages.remove(&key(BUILT)).unwrap();
let moved = "built@git+file:///repos/built#3333333333333333333333333333333333333333";
if let pnpm_lockfile::LockfileResolution::Git(git) = &mut metadata.resolution {
git.commit = "3333333333333333333333333333333333333333".to_string();
}
packages.insert(key(moved), metadata);
let carried = PriorRecords::of(&prior).carried(&lockfile, &key(moved));
assert!(carried.is_none());
}
#[test]
fn applied_records_round_trip_to_the_recorded_lockfile() {
let prior = parse(RECORDED);
let mut lockfile = unrecorded();
let records: HashMap<_, _> = [BUILT, PLAIN]
.into_iter()
.map(|metadata_key| {
let record = PriorRecords::of(&prior)
.carried(&lockfile, &key(metadata_key))
.expect("carried");
(key(metadata_key), record)
})
.collect();
apply_records(&mut lockfile, &records);
assert_eq!(lockfile, prior);
}
/// A package that needs a tool the previous lockfile records reuses that
/// version rather than resolving the tool again.
#[test]
fn recorded_tools_are_offered_for_reuse() {
let prior = parse(RECORDED);
let tools: Vec<String> = PriorRecords::of(&prior)
.tools()
.keys()
.map(ToString::to_string)
.collect();
assert_eq!(tools, ["yarn@tool:1.22.22"]);
}
/// A tool entry no snapshot names is never handed to a new package.
#[test]
fn unreferenced_tool_entries_are_not_offered() {
let mut prior = parse(RECORDED);
for (_, snapshot) in prior.snapshots.iter_mut().flatten() {
snapshot.build_dependencies = None;
}
assert!(PriorRecords::of(&prior).tools().is_empty());
}
/// A carried record whose tool is not that tool's own artifact sends the
/// package back to be examined.
#[test]
fn an_invalid_carried_tool_is_examined_again() {
let prior = parse(&RECORDED.replace(
"tarball: https://registry.npmjs.org/yarn/-/yarn-1.22.22.tgz",
"tarball: 'file:./yarn.tgz'",
));
let carried = PriorRecords::of(&prior).carried(&unrecorded(), &key(BUILT));
assert!(carried.is_none());
}
/// Only a git-hosted package's record offers its tools: one an edited
/// lockfile attaches to a registry package is ignored.
#[test]
fn tools_a_registry_snapshot_names_are_not_offered() {
let mut prior = parse(RECORDED);
let snapshots = prior.snapshots.as_mut().unwrap();
let recorded = snapshots[&key(BUILT)].build_dependencies.clone();
snapshots
.get_mut(&key(BUILT))
.unwrap()
.build_dependencies = None;
snapshots
.get_mut(&key("is-positive@1.0.0"))
.unwrap()
.build_dependencies = recorded;
assert!(PriorRecords::of(&prior).tools().is_empty());
}
@@ -0,0 +1,164 @@
//! Resolve the tools a build environment asks for into lockfile entries.
use super::{BuildToolRecord, BuildToolRecorder, RecordBuildToolsError, ToolCatalog};
use pnpm_git_fetcher::{BuildEnvironment, PreferredPm};
use pnpm_lockfile::{PackageKey, PackageMetadata, Prefix};
use pnpm_resolving_resolver_base::{ResolveResult, WantedDependency};
use pnpm_tool_protocol::{PackageManagerSource, TOOL_PROTOCOL, package_manager_source};
/// A tool a build needs: its name in the lockfile and the range it must
/// satisfy.
struct WantedTool<'a> {
name: &'static str,
version_spec: &'a str,
}
impl WantedTool<'_> {
/// The npm package the tool is published as, or `None` for one that
/// ships as platform archives (Node.js, Bun, Yarn 6 and later).
fn registry_package(&self) -> Option<&'static str> {
match package_manager_source(self.name, self.version_spec)? {
PackageManagerSource::Registry(package) => Some(package),
PackageManagerSource::Archive => None,
}
}
fn describe(&self) -> String {
format!("{}@{}", self.name, self.version_spec)
}
}
impl BuildToolRecorder<'_> {
/// The record of `environment`'s tools. A tool `catalog` already holds
/// at a satisfying version is reused, the way a regular dependency's
/// locked version is. A tool resolved here joins `catalog`.
pub(super) async fn resolve_tools(
&self,
catalog: &mut ToolCatalog,
environment: &BuildEnvironment,
) -> Result<BuildToolRecord, RecordBuildToolsError> {
let mut record = BuildToolRecord::new();
for tool in wanted_tools(environment) {
let entry = if let Some(entry) = locked_tool(catalog, &tool) {
entry
} else {
let entry = self.resolve_tool(&tool).await?;
catalog.insert(entry.0.clone(), entry.1.clone());
entry
};
let alias = tool.name.parse().expect("a tool name is a valid package name");
record.insert(alias, entry);
}
Ok(record)
}
async fn resolve_tool(
&self,
tool: &WantedTool<'_>,
) -> Result<(PackageKey, PackageMetadata), RecordBuildToolsError> {
let registry_package = tool.registry_package();
let wanted = match registry_package {
Some(package) => WantedDependency {
alias: Some(package.to_string()),
bare_specifier: Some(tool.version_spec.to_string()),
..WantedDependency::default()
},
None => WantedDependency {
alias: Some(tool.name.to_string()),
bare_specifier: Some(format!("{TOOL_PROTOCOL}{}", tool.version_spec)),
..WantedDependency::default()
},
};
let resolve_error =
|reason: String| RecordBuildToolsError::Resolve { tool: tool.describe(), reason };
let result = self.resolver
.resolve(&wanted, self.resolve_options)
.await
.map_err(|error| resolve_error(error.to_string()))?
.ok_or_else(|| RecordBuildToolsError::NoMatch { tool: tool.describe() })?;
if let Some(violation) = &result.policy_violation {
return Err(resolve_error(violation.reason.clone()));
}
let version = resolved_version(&result, registry_package)
.ok_or_else(|| RecordBuildToolsError::NoMatch { tool: tool.describe() })?;
let key = format!("{}@{TOOL_PROTOCOL}{version}", tool.name)
.parse()
.map_err(|_| RecordBuildToolsError::NoMatch { tool: tool.describe() })?;
Ok((key, tool_metadata(result)))
}
}
/// The tools `environment` asks for. pnpm is not one of them: the running
/// pnpm, or the version the package pins, prepares the package as it
/// does without a record.
fn wanted_tools(environment: &BuildEnvironment) -> Vec<WantedTool<'_>> {
let package_manager = &environment.package_manager;
let package_manager = (package_manager.pm != PreferredPm::Pnpm).then(|| WantedTool {
name: package_manager.pm.name(),
version_spec: package_manager.version_spec.as_deref().unwrap_or("latest"),
});
let node = environment.node
.as_deref()
.map(|version_spec| WantedTool { name: "node", version_spec });
package_manager
.into_iter()
.chain(node)
.collect()
}
/// The highest version of `tool` that `catalog` holds and the tool's range
/// admits, from the same source the range selects.
fn locked_tool(
catalog: &ToolCatalog,
tool: &WantedTool<'_>,
) -> Option<(PackageKey, PackageMetadata)> {
let range = node_semver::Range::parse(tool.version_spec).ok()?;
let source = package_manager_source(tool.name, tool.version_spec);
catalog
.iter()
.filter(|(key, _)| key.suffix.prefix() == Prefix::Tool && key.name.to_string() == tool.name)
.filter_map(|(key, metadata)| Some((key.suffix.version_semver()?, key, metadata)))
.filter(|(version, _, _)| {
version.satisfies(&range)
&& package_manager_source(tool.name, &version.to_string()) == source
})
.max_by(|(left, _, _), (right, _, _)| left.cmp(right))
.map(|(_, key, metadata)| (key.clone(), metadata.clone()))
}
/// The exact version `result` resolved to. A registry package must be the
/// one asked for: an alias resolving to another package is no answer.
fn resolved_version(result: &ResolveResult, registry_package: Option<&str>) -> Option<String> {
if let Some(package) = registry_package {
let name_ver = result.package.name_ver.as_ref()?;
return (name_ver.name.to_string() == package).then(|| name_ver.suffix.to_string());
}
let (_, version) = result.id.as_str().rsplit_once('@')?;
Some(pnpm_tool_protocol::strip_tool_protocol(version).unwrap_or(version).to_string())
}
fn tool_metadata(result: ResolveResult) -> PackageMetadata {
tool_metadata_of(result.resolution)
}
/// A tool's `packages:` entry: its resolution, and nothing a package's
/// manifest would contribute.
fn tool_metadata_of(resolution: pnpm_lockfile::LockfileResolution) -> PackageMetadata {
PackageMetadata {
resolution,
version: None,
engines: None,
cpu: None,
os: None,
libc: None,
deprecated: None,
has_bin: None,
prepare: None,
bundled_dependencies: None,
peer_dependencies: None,
peer_dependencies_meta: None,
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,32 @@
use super::{WantedTool, locked_tool};
use crate::build_tools_record::ToolCatalog;
use pnpm_lockfile::PackageKey;
fn catalog(keys: &[&str]) -> ToolCatalog {
keys.iter()
.map(|key| {
let text = format!(
"{{integrity: sha512-mQ==, tarball: https://registry.npmjs.org/x/-/{key}.tgz}}",
);
let resolution = serde_saphyr::from_str(&text).expect("parse resolution");
let metadata = super::tool_metadata_of(resolution);
(key.parse::<PackageKey>().expect("parse key"), metadata)
})
.collect()
}
fn locked(catalog: &ToolCatalog, name: &'static str, version_spec: &str) -> Option<String> {
locked_tool(catalog, &WantedTool { name, version_spec }).map(|(key, _)| key.to_string())
}
#[test]
fn the_highest_satisfying_recorded_version_is_reused() {
let catalog =
catalog(&["yarn@tool:1.22.19", "yarn@tool:1.22.22", "yarn@tool:4.9.2", "npm@tool:10.9.2"]);
assert_eq!(locked(&catalog, "yarn", "1"), Some("yarn@tool:1.22.22".to_string()));
assert_eq!(locked(&catalog, "yarn", ">=2"), Some("yarn@tool:4.9.2".to_string()));
assert_eq!(locked(&catalog, "npm", "^10"), Some("npm@tool:10.9.2".to_string()));
assert_eq!(locked(&catalog, "npm", "^11"), None);
// A dist-tag commits to no version, so it resolves.
assert_eq!(locked(&catalog, "npm", "latest"), None);
}
@@ -350,6 +350,7 @@ pub(super) fn build_snapshot_entry(
.unwrap_or(node.optional);
SnapshotEntry {
build_dependencies: None,
id: None,
dependencies: (!dependencies.is_empty()).then_some(dependencies),
optional_dependencies: (!optional_dependencies.is_empty()).then_some(optional_dependencies),
@@ -72,6 +72,7 @@ fn single_importer_opts<'a>(
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers,
dedupe_peers: None,
exclude_links_from_lockfile,
@@ -48,6 +48,7 @@ fn aliased_catalog_dependency_records_catalog_snapshot() {
catalogs: &catalogs,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -527,6 +527,7 @@ fn multi_importer_pruner_marks_shared_dep_non_optional_when_any_importer_reaches
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -620,6 +621,7 @@ fn workspace_sibling_link_renders_per_importer_with_link_ref() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -156,6 +156,7 @@ fn dedupe_peers_round_trips_through_lockfile_settings() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: Some(true),
exclude_links_from_lockfile: false,
@@ -202,6 +203,7 @@ fn dedupe_peers_round_trips_through_lockfile_settings() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -300,6 +302,7 @@ fn patched_dependencies_flow_into_lockfile_and_empty_is_omitted() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -408,6 +411,7 @@ fn snapshot_link_uses_lockfile_root_while_importer_link_uses_project_root() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -518,6 +522,7 @@ fn multi_importer_workspace_writes_per_project_lockfile_entries() {
catalogs: &EMPTY_CATALOGS,
time: BTreeMap::new(),
settings: pnpm_lockfile::LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -193,12 +193,18 @@ fn importer_root_keys(lockfile: &Lockfile) -> VecDeque<PkgNameVerPeer> {
.collect()
}
/// Every edge of `snapshot`, its build tools included: a tool stays in the
/// lockfile while a package that builds with it does.
fn snapshot_child_keys(snapshot: &SnapshotEntry) -> impl Iterator<Item = PkgNameVerPeer> + '_ {
[snapshot.dependencies.as_ref(), snapshot.optional_dependencies.as_ref()]
.into_iter()
.flatten()
.flatten()
.filter_map(|(alias, dep_ref)| dep_ref.resolve(alias))
[
snapshot.dependencies.as_ref(),
snapshot.optional_dependencies.as_ref(),
snapshot.build_dependencies.as_ref(),
]
.into_iter()
.flatten()
.flatten()
.filter_map(|(alias, dep_ref)| dep_ref.resolve(alias))
}
/// Recompute every snapshot's `optional` flag from what still reaches it:
@@ -14,6 +14,7 @@ use std::{
/// resolution.
pub(crate) fn lockfile_settings_from_config(config: &Config) -> LockfileSettings {
LockfileSettings {
lock_build_tools: config.lock_build_tools.then_some(true),
auto_install_peers: config.auto_install_peers,
dedupe_peers: config.dedupe_peers.then_some(true),
exclude_links_from_lockfile: config.exclude_links_from_lockfile,
@@ -54,6 +55,7 @@ pub(crate) enum ChangedSetting {
AddMissingPeerTypes,
PeersSuffixMaxLength,
InjectWorkspacePackages,
LockBuildTools,
ResolutionSettings,
}
@@ -96,17 +98,19 @@ pub(crate) fn apply_settings_update(
true
}
/// Reads one boolean setting the lockfile records.
type RecordedFlag = fn(Option<&LockfileSettings>) -> bool;
fn changed_settings(
recorded: Option<&LockfileSettings>,
settings: &LockfileSettings,
) -> Vec<ChangedSetting> {
let flag_changed = |read: RecordedFlag| read(recorded) != read(Some(settings));
let mut changed = Vec::new();
if pnpm_lockfile::auto_install_peers_changed(recorded, settings.auto_install_peers) {
changed.push(ChangedSetting::AutoInstallPeers);
}
if pnpm_lockfile::recorded_dedupe_peers(recorded)
!= pnpm_lockfile::recorded_dedupe_peers(Some(settings))
{
if flag_changed(pnpm_lockfile::recorded_dedupe_peers) {
changed.push(ChangedSetting::DedupePeers);
}
if pnpm_lockfile::exclude_links_from_lockfile_changed(
@@ -115,9 +119,7 @@ fn changed_settings(
) {
changed.push(ChangedSetting::ExcludeLinksFromLockfile);
}
if pnpm_lockfile::recorded_add_missing_peer_types(recorded)
!= pnpm_lockfile::recorded_add_missing_peer_types(Some(settings))
{
if flag_changed(pnpm_lockfile::recorded_add_missing_peer_types) {
changed.push(ChangedSetting::AddMissingPeerTypes);
}
if pnpm_lockfile::recorded_peers_suffix_max_length(recorded)
@@ -125,11 +127,12 @@ fn changed_settings(
{
changed.push(ChangedSetting::PeersSuffixMaxLength);
}
if pnpm_lockfile::recorded_inject_workspace_packages(recorded)
!= pnpm_lockfile::recorded_inject_workspace_packages(Some(settings))
{
if flag_changed(pnpm_lockfile::recorded_inject_workspace_packages) {
changed.push(ChangedSetting::InjectWorkspacePackages);
}
if flag_changed(pnpm_lockfile::recorded_lock_build_tools) {
changed.push(ChangedSetting::LockBuildTools);
}
// Unrecorded settings are dropped on write rather than compared.
if settings.resolution != ResolutionSettings::default()
&& recorded.map(|recorded| &recorded.resolution) != Some(&settings.resolution)
@@ -156,11 +159,21 @@ fn setting_cannot_affect_lockfile(
ChangedSetting::InjectWorkspacePackages => {
has_no_injectable_dependencies(lockfile, manifests, workspace_package_names)
}
ChangedSetting::LockBuildTools => has_no_git_hosted_packages(lockfile),
// Only a resolution under the recorded values may record them.
ChangedSetting::ResolutionSettings => false,
}
}
/// `lockBuildTools` only records the tools that prepare git-hosted
/// packages.
fn has_no_git_hosted_packages(lockfile: &Lockfile) -> bool {
lockfile.packages
.iter()
.flatten()
.all(|(_, metadata)| !metadata.resolution.is_git_hosted())
}
/// The peer settings only change how peer dependencies are
/// resolved, deduplicated, and hashed into depPath suffixes. None of
/// them has anything to act on when no package or project declares a
@@ -49,6 +49,7 @@ fn manifest(value: Value) -> PackageManifest {
fn recorded_settings() -> LockfileSettings {
LockfileSettings {
lock_build_tools: None,
auto_install_peers: true,
dedupe_peers: None,
exclude_links_from_lockfile: false,
@@ -63,6 +64,7 @@ fn recorded_settings() -> LockfileSettings {
fn records_every_setting_the_locked_graph_cannot_notice() {
let manifest = manifest(json!({ "dependencies": { "foo": "^1.0.0" } }));
let settings = LockfileSettings {
lock_build_tools: None,
auto_install_peers: false,
dedupe_peers: Some(true),
exclude_links_from_lockfile: true,
@@ -515,6 +515,7 @@ pub(crate) fn check_lockfile_settings_drift(
package_extensions_checksum: package_extensions_checksum.as_deref(),
ignored_optional_dependencies: config.ignored_optional_dependencies.as_deref(),
patched_dependencies: patched_dependency_hashes.as_ref(),
lock_build_tools: config.lock_build_tools,
resolution: pnpm_lockfile::ResolutionSettingsCheck {
auto_install_peers: config.auto_install_peers,
dedupe_peers,
@@ -33,6 +33,7 @@ use resolution::{ManifestSlots, Resolved, resolve_graph, start_early_materializa
mod setup;
use setup::{InstallShape, ResolverSetup, set_up_resolvers};
mod build_tools_phase;
mod materialization;
use materialization::finish_resolved_install;
@@ -0,0 +1,77 @@
//! The `lockBuildTools` pass of a fresh-lockfile install: record the tools
//! that prepare each git-hosted package before the lockfile is materialized
//! or saved.
use super::{
FreshInputs,
errors::InstallWithFreshLockfileError,
resolve::SharedResolveOptions,
setup::{ObserverSettings, TrustGate},
};
use crate::{
build_tools_record::{BuildToolRecorder, PackageSources},
resolution_policy::PickPolicy,
};
use pnpm_lockfile::Lockfile;
use pnpm_reporter::Reporter;
use pnpm_resolving_resolver_base::{ResolveOptions, Resolver, UpdateBehavior};
use std::sync::Arc;
/// The install's resolver chain, kept past resolution for the tools.
pub(super) struct BuildToolResolution {
resolver: Box<dyn Resolver>,
options: ResolveOptions,
}
impl BuildToolResolution {
/// Keep `resolver` when `lockBuildTools` is on, to resolve the tools
/// once the lockfile is built, under the release-age and trust policies
/// the dependencies resolved under. Otherwise drop it with the rest of
/// resolution.
pub(super) fn keep(
install: FreshInputs<'_>,
(policy, observer): (&PickPolicy, &ObserverSettings),
resolver: Box<dyn Resolver>,
) -> Result<Option<Self>, InstallWithFreshLockfileError> {
let config = install.drivers.config;
if !config.lock_build_tools {
return Ok(None);
}
let trust = TrustGate::of(config)?;
let lockfile_dir = install.projects.lockfile_dir;
let options = SharedResolveOptions {
policy: pnpm_resolving_resolver_base::ResolutionPolicyOptions {
published_by: policy.published_by,
published_by_exclude: policy.published_by_exclude.clone(),
trust_policy: trust.policy,
trust_policy_exclude: trust.exclude,
package_version_guard: observer.package_version_guard.clone(),
..Default::default()
},
config,
lockfile_dir,
workspace_packages: None,
update_checksums: install.update_checksums,
update_behavior: UpdateBehavior::Off,
}
.build(lockfile_dir.to_path_buf(), Arc::default());
Ok(Some(Self { resolver, options }))
}
/// Record the tools in `lockfile`, carrying records over from `prior`.
pub(super) async fn record<Reporter: self::Reporter>(
self,
lockfile: &mut Lockfile,
prior: Option<&Lockfile>,
sources: PackageSources<'_>,
) -> Result<(), InstallWithFreshLockfileError> {
BuildToolRecorder {
resolver: self.resolver.as_ref(),
resolve_options: &self.options,
sources,
}
.record::<Reporter>(lockfile, prior)
.await
.map_err(InstallWithFreshLockfileError::RecordBuildTools)
}
}
@@ -109,6 +109,9 @@ pub enum InstallWithFreshLockfileError {
#[diagnostic(transparent)]
RootDepManifest(#[error(source)] pnpm_package_manifest::PackageManifestError),
#[diagnostic(transparent)]
RecordBuildTools(#[error(source)] crate::RecordBuildToolsError),
#[display("Failed to build lockfile from resolved dependency graph: {_0}")]
#[diagnostic(code(pnpm_package_manager::dependencies_graph_to_lockfile))]
DependenciesGraphToLockfile(#[error(source)] Box<DependenciesGraphToLockfileError>),
@@ -2,9 +2,10 @@ use super::{
FinalScope, FreshInputs, FreshPlan, HostProbeInputs, InstallShape,
InstallWithFreshLockfileResult, LockfileOnlyOptions, LockfileViews, MaterializationScope,
OnDiskInputs, OnDiskOutput, OwnedInputs, PlanLockfiles, PlanScope, Resolved, ResolverSetup,
build_lockfile_phase, errors::InstallWithFreshLockfileError, finish_early_materialization,
finish_lockfile_only, manifest_transforms, persist_fresh_lockfile, plan_fresh_materialization,
resolver_setup, run_on_disk_phases, warn_stale_convergence_overrides_if_any,
build_lockfile_phase, build_tools_phase::BuildToolResolution,
errors::InstallWithFreshLockfileError, finish_early_materialization, finish_lockfile_only,
manifest_transforms, persist_fresh_lockfile, plan_fresh_materialization, resolver_setup,
run_on_disk_phases, warn_stale_convergence_overrides_if_any,
};
use crate::{AllowBuildPolicy, VirtualStoreLayout};
use pnpm_catalogs_types::Catalogs;
@@ -51,6 +52,7 @@ pub(super) struct FreshMaterialization<'a, Reporter> {
shape: InstallShape,
stores: MaterializationStores,
custom_fetcher_session: Option<Arc<pnpm_deps_restorer::CustomFetcherSession>>,
build_tools: Option<BuildToolResolution>,
resolved: Resolved<'a, Reporter>,
}
/// Release resolution caches before materializing. Boxing this phase bounds the
@@ -66,19 +68,19 @@ pub(super) fn finish_resolved_install<'a, Reporter: self::Reporter + 'static>(
> {
Box::pin(async move {
warn_stale_overrides::<Reporter>(install, &setup, &resolved).await;
let build_tools = BuildToolResolution::keep(
install,
(&setup.policy, &setup.observer),
setup.chain.resolver,
)?;
// Resolution is over: release the resolvers and the metadata
// cache before materializing, so their memory does not outlive
// its use. The custom fetchers stay, the cold batch consults them.
drop(setup.chain.resolver);
drop(setup.chain.npm_resolver);
drop(owned.fetching.meta_cache);
drop(setup.chain.fetch_locker);
drop(setup.chain.picked_manifest_cache);
Reporter::emit(&LogEvent::Stage(StageLog {
level: LogLevel::Debug,
prefix: install.projects.lockfile_dir.display().to_string(),
stage: Stage::ResolutionDone,
}));
emit_resolution_done::<Reporter>(install);
Box::pin(
FreshMaterialization {
install,
@@ -99,6 +101,7 @@ pub(super) fn finish_resolved_install<'a, Reporter: self::Reporter + 'static>(
shape: setup.shape,
stores: setup.stores.into(),
custom_fetcher_session: setup.chain.custom_fetcher_session,
build_tools,
resolved,
}
.run(),
@@ -106,6 +109,13 @@ pub(super) fn finish_resolved_install<'a, Reporter: self::Reporter + 'static>(
.await
})
}
fn emit_resolution_done<Reporter: self::Reporter>(install: FreshInputs<'_>) {
Reporter::emit(&LogEvent::Stage(StageLog {
level: LogLevel::Debug,
prefix: install.projects.lockfile_dir.display().to_string(),
stage: Stage::ResolutionDone,
}));
}
pub(super) async fn warn_stale_overrides<Reporter: self::Reporter + 'static>(
install: FreshInputs<'_>,
setup: &ResolverSetup,
@@ -150,7 +160,7 @@ impl<Reporter: self::Reporter + 'static> FreshMaterialization<'_, Reporter> {
.then(|| AllowBuildPolicy::from_config(self.install.drivers.config))
.transpose()
.map_err(InstallWithFreshLockfileError::AllowBuildsPolicy)?;
let built_lockfile = build_lockfile_phase::<Reporter>(
let mut built_lockfile = build_lockfile_phase::<Reporter>(
self.install,
&mut self.resources.lockfile_verification_gate,
std::mem::take(&mut self.resolved.graph.time),
@@ -168,12 +178,31 @@ impl<Reporter: self::Reporter + 'static> FreshMaterialization<'_, Reporter> {
self.shape.verify_filtered_repair,
)
.await?;
self.record_build_tools(&mut built_lockfile).await?;
match allow_build_policy {
Some(policy) => self.materialize(built_lockfile, policy).await,
None => self.finish_lockfile_only(built_lockfile).await,
}
}
async fn record_build_tools(
&mut self,
lockfile: &mut Lockfile,
) -> Result<(), InstallWithFreshLockfileError> {
let Some(build_tools) = self.build_tools.take() else { return Ok(()) };
let prior = self.resolved.fixed_wanted_lockfile
.as_ref()
.or(self.install.lockfiles.wanted);
let sources = crate::build_tools_record::PackageSources {
config: self.install.drivers.config,
http_client: self.install.drivers.http_client,
git_source_cache: &self.stores.caches.git_source_cache,
tarball_mem_cache: &self.resources.tarball_mem_cache,
store_index: self.stores.index.as_ref(),
};
build_tools.record::<Reporter>(lockfile, prior, sources).await
}
async fn finish_lockfile_only(
self,
built_lockfile: Lockfile,
@@ -286,7 +286,7 @@ pub(super) struct TrustGate {
pub(super) exclude: Option<pnpm_config::version_policy::PackageVersionPolicy>,
}
impl TrustGate {
fn of(config: &Config) -> Result<Self, InstallWithFreshLockfileError> {
pub(super) fn of(config: &Config) -> Result<Self, InstallWithFreshLockfileError> {
Ok(Self {
policy: resolver_trust_policy(config.trust_policy),
exclude: config.trust_policy_exclude
+2
View File
@@ -7,6 +7,7 @@ pub use agent_skills::{
};
pub use build_resolution_verifiers::*;
pub use build_snapshot::*;
pub use build_tools_record::RecordBuildToolsError;
pub use catalog_cleanup::{WriteWorkspaceCatalogsError, prune_against_project_lockfiles};
pub use catalog_mode::*;
pub use command_lockfile::CommandLockfile;
@@ -48,6 +49,7 @@ mod add;
mod agent_skills;
mod build_resolution_verifiers;
mod build_snapshot;
mod build_tools_record;
mod catalog_cleanup;
mod catalog_mode;
mod check_custom_resolver_force_resolve;
@@ -119,6 +119,15 @@ impl SettingsComparison<'_> {
.or_else(|| self.resolution_drift())
.or_else(|| self.workspace_policy_drift())
.or_else(|| self.agent_skills_drift())
.or_else(|| self.build_tools_drift())
}
fn build_tools_drift(&self) -> Option<&'static str> {
return_drift_if!(
self,
"lockBuildTools",
self.recorded.lock_build_tools != self.live.lock_build_tools,
);
None
}
fn installation_drift(&self) -> Option<&'static str> {
let recorded = self.recorded;
@@ -354,6 +363,7 @@ fn recorded_allow_builds(
fn current_policy_settings(config: &Config) -> WorkspaceStateSettings {
WorkspaceStateSettings {
lock_build_tools: config.lock_build_tools.then_some(true),
lockfile_include_resolution_settings: config
.lockfile_include_resolution_settings
.then_some(true),
+2 -1
View File
@@ -333,7 +333,7 @@ async fn git_hosted_cas_paths<Reporter: self::Reporter>(
return Ok(cas_paths);
}
let allow_build_closure = |_dep_path: &str| Some(false);
let files_index_file = git_hosted_store_index_key(package_id, !config.ignore_scripts);
let files_index_file = git_hosted_store_index_key(package_id, !config.ignore_scripts, None);
let GitFetchOutput { cas_paths, built: _built } = GitHostedTarballFetcher {
scripts: pnpm_git_fetcher::PrepareScriptOptions {
ignore: config.ignore_scripts,
@@ -358,6 +358,7 @@ async fn git_hosted_cas_paths<Reporter: self::Reporter>(
// manager has to be provided to it.
package_id,
requester: "",
build_tools: pnpm_git_fetcher::BuildTools::Detected,
}
.run::<Reporter>()
.await
+2 -2
View File
@@ -14,8 +14,8 @@ pub use project::{
pub use readme::{ReadmeKind, decode_readme, is_preferred_readme, readme_kind};
pub use runtime::{
apply_runtime_on_fail_override, convert_dependencies_to_engines_runtime,
convert_engines_runtime_to_dependencies, engines_runtime_dependencies, is_runtime_alias,
node_version_from_engines_runtime,
convert_engines_runtime_to_dependencies, downloadable_runtime_version,
engines_runtime_dependencies, is_runtime_alias, node_version_from_engines_runtime,
};
pub use serialization::{parse_manifest, parse_manifest_bytes, safe_read_package_json_from_dir};
pub use truthiness::is_truthy;
+34 -33
View File
@@ -56,41 +56,42 @@ pub fn engines_runtime_dependencies(
engines_field: &str,
deps_field: &str,
) -> Vec<(&'static str, String)> {
let mut dependencies = Vec::new();
let Some(runtime_entry) = manifest
.get(engines_field)
.and_then(|engines| engines.get("runtime"))
else {
return dependencies;
RUNTIME_NAMES
.into_iter()
.filter(|runtime_name| {
manifest
.get(deps_field)
.and_then(|deps| deps.get(runtime_name))
.is_none()
})
.filter_map(|runtime_name| {
let version = downloadable_runtime_version(manifest, engines_field, runtime_name)?;
Some((runtime_name, format!("runtime:{version}")))
})
.collect()
}
/// The version the first `runtime_name` entry of
/// `manifest[engines_field].runtime` pins, when that entry asks for
/// `onFail: "download"`.
#[must_use]
pub fn downloadable_runtime_version<'a>(
manifest: &'a Value,
engines_field: &str,
runtime_name: &str,
) -> Option<&'a str> {
let runtimes: &[Value] = match manifest.get(engines_field)?.get("runtime")? {
Value::Array(arr) => arr.as_slice(),
single @ Value::Object(_) => std::slice::from_ref(single),
_ => return None,
};
for runtime_name in RUNTIME_NAMES {
if manifest
.get(deps_field)
.and_then(|deps| deps.get(runtime_name))
.is_some()
{
continue;
}
let runtimes: &[Value] = match runtime_entry {
Value::Array(arr) => arr.as_slice(),
single @ Value::Object(_) => std::slice::from_ref(single),
_ => continue,
};
let Some(runtime) = runtimes
.iter()
.find(|runtime| runtime.get("name").and_then(Value::as_str) == Some(runtime_name))
else {
continue;
};
if runtime.get("onFail").and_then(Value::as_str) != Some("download") {
continue;
}
let Some(version) = runtime.get("version").and_then(Value::as_str) else {
continue;
};
dependencies.push((runtime_name, format!("runtime:{}", version.trim())));
let runtime = runtimes
.iter()
.find(|runtime| runtime.get("name").and_then(Value::as_str) == Some(runtime_name))?;
if runtime.get("onFail").and_then(Value::as_str) != Some("download") {
return None;
}
dependencies
Some(runtime.get("version")?.as_str()?.trim())
}
/// Apply the configured runtime failure policy to both engine fields.
@@ -108,7 +108,7 @@ fn keys_that_name_no_package_are_not_checked() {
assert_eq!(
pkg_content_mismatch(
Some(&manifest("bar", "2.0.0")),
&crate::git_hosted_store_index_key("github.com/foo/bar#deadbeef", true),
&crate::git_hosted_store_index_key("github.com/foo/bar#deadbeef", true, None),
),
None,
);
+12 -3
View File
@@ -357,9 +357,18 @@ pub fn store_index_key(integrity: &str, pkg_id: &str) -> String {
/// ran during fetch (`preparePackage`), so `built` is part of the key. The
/// integrity-only key would collapse the built/not-built variants into one
/// slot.
///
/// `build_tools` identifies the tools the lockfile records for the build
/// (`lockBuildTools`), so a package built by one set of tools is never
/// served to an install that records another. It takes no part in the key
/// of a package that was not built.
#[must_use]
pub fn git_hosted_store_index_key(pkg_id: &str, built: bool) -> String {
store_index_key(pkg_id, if built { "built" } else { "not-built" })
pub fn git_hosted_store_index_key(pkg_id: &str, built: bool, build_tools: Option<&str>) -> String {
match (built, build_tools) {
(true, Some(build_tools)) => store_index_key(pkg_id, &format!("built:{build_tools}")),
(true, None) => store_index_key(pkg_id, "built"),
(false, _) => store_index_key(pkg_id, "not-built"),
}
}
/// Pick the store-index key for a tarball-shaped resolution.
@@ -375,7 +384,7 @@ pub fn pick_store_index_key(
) -> String {
match integrity {
Some(integrity) if !git_hosted => store_index_key(integrity, pkg_id),
_ => git_hosted_store_index_key(pkg_id, built),
_ => git_hosted_store_index_key(pkg_id, built, None),
}
}
@@ -66,11 +66,11 @@ fn key_format_is_integrity_tab_pkg_id() {
fn git_hosted_key_uses_built_marker() {
// Mirrors upstream's `gitHostedStoreIndexKey(pkgId, { built })`.
assert_eq!(
git_hosted_store_index_key("github.com/foo/bar/abc1234", true),
git_hosted_store_index_key("github.com/foo/bar/abc1234", true, None),
"github.com/foo/bar/abc1234\tbuilt",
);
assert_eq!(
git_hosted_store_index_key("github.com/foo/bar/abc1234", false),
git_hosted_store_index_key("github.com/foo/bar/abc1234", false, None),
"github.com/foo/bar/abc1234\tnot-built",
);
}
+1
View File
@@ -11,6 +11,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
node-semver = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
Loaded 100 of 108 files, more files were not shown because too many files have changed in this diff. Show more