feat: pin shared build artifacts in the lockfile (#14243)

Persist verified shared build artifact envelope digests in the package snapshot
that consumed them. Scope each pin by dependency input key, artifact owner, and
consumer platform so compatible variants remain independently reproducible.

Apply pins only after the existing trust and compatibility checks. A missing
pinned artifact is a remote-cache miss and falls back to a local build; frozen
installs enforce pins without mutating the wanted lockfile.

Add `pnpm update --build-artifacts` as an explicit update mode that clears
existing pins, forces materialization, and records only artifacts that complete
verification and hydration without changing package versions. Reject command
combinations that cannot safely write or verify the lockfile.

Implement the lockfile format, install plumbing, update mode, and tests in both
TypeScript pnpm and pacquet.

Related to pnpm/pnpm#13771.
This commit is contained in:
Zoltan Kochan authored and GitHub committed 2026-08-27 23:08:19 +02:00
1 parent aaaca889de
commit 5dd145394b
39 files changed
+992 -100

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
---
"@pnpm/lockfile.types": minor
"@pnpm/lockfile.fs": patch
"@pnpm/pnpr.client": minor
"@pnpm/installing.deps-resolver": patch
"@pnpm/installing.deps-restorer": minor
"@pnpm/installing.deps-installer": minor
"@pnpm/installing.commands": minor
"pnpm": minor
"pacquet": minor
---
Remote build artifacts are now pinned in `pnpm-lock.yaml` by input key, owner, and consumer platform. Frozen installs enforce existing pins without changing them. Run `pnpm update --build-artifacts` to explicitly replace the recorded pins ([pnpm/pnpm#13771](https://github.com/pnpm/pnpm/issues/13771)).
+65 -29
View File
@@ -244,6 +244,9 @@ pub struct InstallArgs {
#[clap(long = "update-checksums")]
pub update_checksums: bool,
#[clap(skip)]
pub(crate) refresh_artifact_pins: bool,
/// Maximum number of concurrent network requests during install.
#[clap(long = "network-concurrency")]
pub network_concurrency: Option<usize>,
@@ -327,6 +330,7 @@ impl InstallArgs {
trust_lockfile: false,
no_trust_lockfile: false,
update_checksums: false,
refresh_artifact_pins: false,
network_concurrency: None,
fetch_timeout: None,
fetch_warn_timeout_ms: None,
@@ -336,6 +340,16 @@ impl InstallArgs {
}
}
pub(crate) fn for_build_artifact_update(
supported_architectures: SupportedArchitecturesArgs,
) -> Self {
let mut args = Self::for_reresolving_install();
args.supported_architectures = supported_architectures;
args.force = true;
args.refresh_artifact_pins = true;
args
}
/// Run the repeat-install fast path before any of the async install
/// machinery exists: when every gate below holds and
/// [`install_already_up_to_date`] confirms nothing changed since the
@@ -366,6 +380,7 @@ impl InstallArgs {
if self.effective_frozen_lockfile(config)
|| self.lockfile_only
|| self.force
|| self.refresh_artifact_pins
|| self.verify_deps_before_run_install
{
return false;
@@ -495,6 +510,7 @@ impl InstallArgs {
) -> miette::Result<()> {
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
let frozen_lockfile = match self.configured_frozen_lockfile(state.config) {
_ if self.refresh_artifact_pins => false,
Some(value) => value,
None if state.config.ci
&& !self.lockfile_only
@@ -550,6 +566,7 @@ impl InstallArgs {
trust_lockfile,
no_trust_lockfile,
update_checksums,
refresh_artifact_pins,
network_concurrency: _,
fetch_timeout: _,
fetch_warn_timeout_ms: _,
@@ -566,7 +583,9 @@ impl InstallArgs {
// mutual `overrides_with` collapses both spellings to the
// last-specified, so at most one is set and the precedence here
// is straightforward.
let prefer_frozen_lockfile = if prefer_frozen_lockfile {
let prefer_frozen_lockfile = if refresh_artifact_pins {
Some(false)
} else if prefer_frozen_lockfile {
Some(true)
} else if no_prefer_frozen_lockfile {
Some(false)
@@ -607,7 +626,8 @@ impl InstallArgs {
// pnpr fast path: when a `pnprServer` URL is configured, offload
// resolution + fetching to it, then link `node_modules` from the
// server-produced lockfile via the normal frozen install.
if let Some(pnpr_server) = config.pnpr_server.as_deref() {
if let Some(pnpr_server) = config.pnpr_server.as_deref().filter(|_| !refresh_artifact_pins)
{
// The pnpr path resolves and links through the server, so it
// can't honor `--dry-run`'s no-write contract. Reject up front,
// mirroring pnpm's CONFIG_CONFLICT_DRY_RUN_WITH_PNPR_SERVER.
@@ -639,6 +659,28 @@ impl InstallArgs {
.await;
}
if refresh_artifact_pins && !config.lockfile {
return Err(BuildArtifactsWithoutLockfile.into());
}
let mut refreshed_lockfile = None;
if refresh_artifact_pins {
refreshed_lockfile = lockfile
.get()
.map_err(|error| miette::Report::new(error).wrap_err("load the lockfile"))?
.cloned();
if let Some(snapshots) =
refreshed_lockfile.as_mut().and_then(|lockfile| lockfile.snapshots.as_mut())
{
for snapshot in snapshots.values_mut() {
snapshot.clear_artifact_pins();
}
}
}
let install_lockfile = if refresh_artifact_pins {
MaybeLazyLockfile::Loaded(refreshed_lockfile.as_ref())
} else {
MaybeLazyLockfile::Lazy(lockfile)
};
let install = Install {
tarball_mem_cache: std::sync::Arc::clone(tarball_mem_cache),
http_client,
@@ -646,7 +688,7 @@ impl InstallArgs {
config,
manifest,
emit_initial_manifest: true,
lockfile: MaybeLazyLockfile::Lazy(lockfile),
lockfile: install_lockfile,
lockfile_path: Some(&lockfile_path),
dependency_groups: dependency_options.dependency_groups(config.optional),
frozen_lockfile,
@@ -773,6 +815,11 @@ struct FrozenStoreIncompatibleWithPnpr;
)]
struct DryRunIncompatibleWithPnpr;
#[derive(Debug, Display, Error, Diagnostic)]
#[display("Cannot update build artifacts when lockfile is set to false")]
#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE))]
struct BuildArtifactsWithoutLockfile;
fn resolve_project(
dir: String,
manifest: &pnpm_package_manifest::PackageManifest,
@@ -1119,25 +1166,13 @@ async fn install_via_pnpr_inner<Reporter: self::Reporter + 'static>(
workspace_projects_override: None,
};
let result = match (selection, lockfile_verification_override) {
(Some(selection), Some(lockfile_verification_override)) => {
Box::pin(install.run_selected_with_lockfile_verification::<Reporter>(
workspace_install_selection(selection),
lockfile_verification_override,
))
.await
}
(Some(selection), None) => {
Box::pin(install.run_selected::<Reporter>(workspace_install_selection(selection)))
.await
}
(None, Some(lockfile_verification_override)) => {
install
.run_with_lockfile_verification::<Reporter>(lockfile_verification_override)
.await
}
(None, None) => install.run::<Reporter>().await,
};
let result = install
.run_with_artifact_pin_recording::<Reporter>(
selection.map(workspace_install_selection),
lockfile_verification_override,
!link.frozen_lockfile,
)
.await;
// On failure the prefetcher is dropped, not shut down: shutdown
// waits for every in-flight prefetch download (each task holds a
// store-index writer handle), which would hold the fail-fast
@@ -1353,13 +1388,14 @@ async fn install_via_pnpr_inner<Reporter: self::Reporter + 'static>(
pnpmfile_hook_override: pnpmfile_hook,
workspace_projects_override: None,
};
match selection {
Some(selection) => {
Box::pin(install.run_selected::<Reporter>(workspace_install_selection(selection))).await
}
None => install.run::<Reporter>().await,
}
.wrap_err("linking dependencies resolved via the pnpr server")?;
install
.run_with_artifact_pin_recording::<Reporter>(
selection.map(workspace_install_selection),
None,
!link.frozen_lockfile,
)
.await
.wrap_err("linking dependencies resolved via the pnpr server")?;
// The materialization install has awaited every tarball's mem-cache
// slot, so all prefetch downloads have finished and queued their
+1 -1
View File
@@ -712,7 +712,7 @@ pub(crate) fn apply_install_cli_config(cfg: &mut Config, args: &InstallArgs) {
cfg.ignore_scripts =
resolve_bool_override(args.ignore_scripts, args.no_ignore_scripts, cfg.ignore_scripts);
cfg.ignore_pnpmfile = args.ignore_pnpmfile || cfg.ignore_pnpmfile;
cfg.force = args.force || cfg.force;
cfg.force = args.force || args.refresh_artifact_pins || cfg.force;
if let Some(network_concurrency) = args.network_concurrency {
cfg.network_concurrency = network_concurrency;
}
+49
View File
@@ -1,6 +1,7 @@
use crate::{
State,
cli_args::{
install::InstallArgs,
lockfile_dir::LockfileDirArg,
pipelines::InstallFamilySelection,
recursive,
@@ -94,6 +95,10 @@ pub struct UpdateArgs {
#[clap(long)]
pub patches: bool,
/// Update remote build artifacts without changing package versions.
#[clap(long = "build-artifacts")]
pub build_artifacts: bool,
/// Write the resolved version without a range operator when
/// rewriting the manifest under `--latest`.
#[clap(short = 'E', long = "save-exact")]
@@ -177,9 +182,22 @@ enum WorkspaceUpdateError {
#[diagnostic(code(ERR_PNPM_PATCHES_WITH_SELECTOR))]
struct PatchesWithSelectorError;
#[derive(Debug, Display, Error, Diagnostic)]
#[display(
"--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches"
)]
#[diagnostic(code(ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR))]
struct BuildArtifactsWithSelectorError;
#[derive(Debug, Display, Error, Diagnostic)]
#[display("Cannot update build artifacts without downloading and verifying them")]
#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY))]
struct BuildArtifactsWithLockfileOnlyError;
impl UpdateArgs {
pub(crate) fn apply_cli_config(&self, config: &mut Config) {
config.ignore_pnpmfile = self.ignore_pnpmfile || config.ignore_pnpmfile;
config.force = self.build_artifacts || config.force;
if let Some(pnpr_server) = self.pnpr_server.clone() {
config.pnpr_server = Some(pnpr_server);
}
@@ -190,7 +208,13 @@ impl UpdateArgs {
mut state: State,
) -> miette::Result<()> {
self.check_patches_options()?;
self.check_build_artifacts_options()?;
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
if self.build_artifacts {
return InstallArgs::for_build_artifact_update(self.supported_architectures.clone())
.run::<Reporter>(state)
.await;
}
let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?;
let include_direct = self.dependency_options.include_direct();
let update_actions = self.should_update_github_actions(state.config, &include_direct);
@@ -320,7 +344,13 @@ impl UpdateArgs {
selection: InstallFamilySelection,
) -> miette::Result<()> {
self.check_patches_options()?;
self.check_build_artifacts_options()?;
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
if self.build_artifacts {
return InstallArgs::for_build_artifact_update(self.supported_architectures.clone())
.run_selected::<Reporter>(state, selection)
.await;
}
let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?;
let include_direct = self.dependency_options.include_direct();
let update_actions = self.should_update_github_actions(state.config, &include_direct);
@@ -455,6 +485,7 @@ impl UpdateArgs {
config: &'static Config,
) -> miette::Result<()> {
self.check_patches_options()?;
self.check_build_artifacts_options()?;
self.check_workspace_option(None)?;
let selected_hashes: Option<HashSet<String>> = if self.interactive {
match crate::cli_args::update_interactive::select_global_package_groups(
@@ -515,6 +546,24 @@ impl UpdateArgs {
Ok(())
}
fn check_build_artifacts_options(&self) -> miette::Result<()> {
if !self.build_artifacts {
return Ok(());
}
if !self.packages.is_empty()
|| self.latest
|| self.interactive
|| self.global
|| self.patches
{
return Err(BuildArtifactsWithSelectorError.into());
}
if self.lockfile_only {
return Err(BuildArtifactsWithLockfileOnlyError.into());
}
Ok(())
}
fn can_delegate_patch_refresh(
&self,
update_actions: bool,
@@ -160,6 +160,44 @@ fn patches_is_a_selectorless_update_mode() {
}
}
#[test]
fn build_artifacts_is_a_selectorless_update_mode() {
let build_artifacts = update_args(&["--build-artifacts"]);
assert!(build_artifacts.build_artifacts);
build_artifacts.check_build_artifacts_options().expect("standalone --build-artifacts");
for args in [
&["--build-artifacts", "foo"][..],
&["--build-artifacts", "--latest"][..],
&["--build-artifacts", "--interactive"][..],
&["--build-artifacts", "--global"][..],
&["--build-artifacts", "--patches"][..],
] {
let error = update_args(args)
.check_build_artifacts_options()
.expect_err("--build-artifacts combination must fail");
assert_eq!(
error.to_string(),
"--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches",
);
}
let error = update_args(&["--build-artifacts", "--lockfile-only"])
.check_build_artifacts_options()
.expect_err("--build-artifacts --lockfile-only must fail");
assert_eq!(
error.to_string(),
"Cannot update build artifacts without downloading and verifying them",
);
}
#[test]
fn build_artifacts_forces_reinstallation() {
let mut config = Config::default();
update_args(&["--build-artifacts"]).apply_cli_config(&mut config);
assert!(config.force);
}
#[test]
fn constrained_patch_refresh_stays_on_the_client() {
let all_groups = vec![DependencyGroup::Prod, DependencyGroup::Dev, DependencyGroup::Optional];
+47
View File
@@ -25,6 +25,53 @@ const HAS_PRERELEASE: &str = "@pnpm.e2e/has-prerelease";
/// indirect-dependency update behavior when the direct dep is ignored.
const PARENT: &str = "@pnpm.e2e/pkg-with-1-dep";
#[test]
fn build_artifacts_clears_existing_pins_and_overrides_configured_frozen_lockfile() {
let (root, workspace, npmrc_info) = setup();
let AddMockedRegistry { mock_instance, .. } = npmrc_info;
write_manifest(&workspace, r#"{ "is-positive": "1.0.0" }"#);
pacquet(&workspace, ["install"]).assert().success();
let wanted_path = workspace.join("pnpm-lock.yaml");
let mut wanted = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace)
.expect("load wanted lockfile")
.expect("wanted lockfile");
wanted
.snapshots
.as_mut()
.and_then(|snapshots| snapshots.values_mut().next())
.expect("dependency snapshot")
.record_artifact_pin(
"dependency-side-effects:v1:deps=old".to_string(),
"organization:acme".to_string(),
"platform".to_string(),
"0".repeat(64),
);
wanted.save_to_path(&wanted_path).expect("save artifact pin");
let workspace_yaml_path = workspace.join("pnpm-workspace.yaml");
let mut workspace_yaml =
fs::read_to_string(&workspace_yaml_path).expect("read workspace settings");
workspace_yaml.push_str("frozenLockfile: true\n");
fs::write(workspace_yaml_path, workspace_yaml).expect("write workspace settings");
pacquet(&workspace, ["update", "--build-artifacts"]).assert().success();
let refreshed = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace)
.expect("load refreshed lockfile")
.expect("refreshed lockfile");
assert!(
refreshed
.snapshots
.as_ref()
.into_iter()
.flat_map(|snapshots| snapshots.values())
.all(|snapshot| snapshot.artifact_pins.is_none()),
);
drop((root, mock_instance));
}
/// Spin up a temp workspace with the mocked registry and return the
/// pieces a multi-step update test needs.
fn setup() -> (TempDir, std::path::PathBuf, AddMockedRegistry) {
@@ -35,6 +35,7 @@ fn snap(deps: &[(&str, &str)]) -> SnapshotEntry {
id: None,
dependencies: (!map.is_empty()).then_some(map),
optional_dependencies: None,
artifact_pins: None,
transitive_peer_dependencies: None,
patched: None,
optional: false,
@@ -120,6 +120,16 @@ pub struct CreateVirtualStoreOutput {
/// the bridge into the link phase instead). Pacquet decouples
/// fetch and walk, so the index is built here at fetch time.
pub cas_paths_by_pkg_id: Option<CasPathsByPkgId>,
pub artifact_pin_records: Vec<ArtifactPinRecord>,
}
#[derive(Debug)]
pub struct ArtifactPinRecord {
pub snapshot_key: PackageKey,
pub input_key: String,
pub owner: String,
pub platform_fingerprint: String,
pub envelope_digest: String,
}
/// This subroutine generates filesystem layout for the virtual store at `node_modules/.pacquet`.
@@ -304,6 +314,7 @@ impl CreateVirtualStore<'_> {
materialized_snapshots: Vec::new(),
fetch_failed: HashSet::new(),
cas_paths_by_pkg_id: is_hoisted.then(CasPathsByPkgId::new),
artifact_pin_records: Vec::new(),
});
};
let packages = packages.ok_or(CreateVirtualStoreError::MissingPackagesSection)?;
@@ -856,7 +867,7 @@ impl CreateVirtualStore<'_> {
}
}
crate::shared_side_effects::apply_shared_side_effects(
let artifact_pin_records = crate::shared_side_effects::apply_shared_side_effects(
config,
snapshots,
packages,
@@ -919,6 +930,7 @@ impl CreateVirtualStore<'_> {
materialized_snapshots,
fetch_failed,
cas_paths_by_pkg_id,
artifact_pin_records,
})
}
}
@@ -1,12 +1,13 @@
use crate::{
AllowBuildPolicy, BuildModules, BuildModulesError, CreateVirtualStore, CreateVirtualStoreError,
CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies, LinkHoistedModulesError,
LinkHoistedModulesOpts, LinkRootComponentMembersError, LinkVirtualStoreBinsError,
LockfileToHoistedDepGraphOptions, SkippedSnapshots, SymlinkDirectDependencies,
SymlinkDirectDependenciesError, SymlinkPackageError, VersionPolicyError, VirtualStoreLayout,
any_installability_constraint, build_direct_deps_by_importer, direct_dep_names_for_importer,
get_hoisted_dependencies, link_hoisted_modules, link_top_level_bins,
lockfile_to_hoisted_dep_graph, symlink_direct_dependencies::importer_root_dir,
AllowBuildPolicy, ArtifactPinRecord, BuildModules, BuildModulesError, CreateVirtualStore,
CreateVirtualStoreError, CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies,
LinkHoistedModulesError, LinkHoistedModulesOpts, LinkRootComponentMembersError,
LinkVirtualStoreBinsError, LockfileToHoistedDepGraphOptions, SkippedSnapshots,
SymlinkDirectDependencies, SymlinkDirectDependenciesError, SymlinkPackageError,
VersionPolicyError, VirtualStoreLayout, any_installability_constraint,
build_direct_deps_by_importer, direct_dep_names_for_importer, get_hoisted_dependencies,
link_hoisted_modules, link_top_level_bins, lockfile_to_hoisted_dep_graph,
symlink_direct_dependencies::importer_root_dir,
};
mod build_phase;
@@ -668,6 +669,7 @@ where
materialized_snapshots,
fetch_failed,
cas_paths_by_pkg_id,
artifact_pin_records,
} = {
let mut verify_fut = std::pin::pin!(verify_fut);
let mut create_virtual_store_fut = std::pin::pin!(create_virtual_store_fut);
@@ -879,6 +881,7 @@ where
ignored_builds,
deferred_builds,
store_index_teardown: writer_task,
artifact_pin_records,
})
}
}
@@ -889,6 +892,7 @@ where
/// land without churning every call site.
#[derive(Debug)]
pub struct InstallFrozenLockfileOutput {
pub artifact_pin_records: Vec<ArtifactPinRecord>,
/// Hoisted-dependencies map produced by the isolated-linker
/// hoist pass — empty when both hoist patterns are `None` and
/// always empty under `nodeLinker: hoisted` (the hoisted
@@ -1,6 +1,7 @@
use crate::{
AllowBuildPolicy, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot, build_deps_subgraph,
deps_graph::in_lockfile_order, install_frozen_lockfile::find_runtime_node_major,
AllowBuildPolicy, ArtifactPinRecord, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot,
build_deps_subgraph, deps_graph::in_lockfile_order,
install_frozen_lockfile::find_runtime_node_major,
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use pnpm_config::Config;
@@ -10,7 +11,7 @@ use pnpm_pnpr_client::{
ArtifactManifest, ArtifactPayload, BuilderProfile, CompatibilityConstraints,
LinuxGlibcPlatform, OwnerScope, PackageIdentity, PnprClient, PublishArtifactRequest,
ResolveArtifactsOptions, SignedArtifactEnvelope, blob_id, linux_glibc_supported_tags,
linux_glibc_tag,
linux_glibc_tag, platform_fingerprint,
};
use sha2::{Digest as _, Sha512};
use std::{
@@ -36,6 +37,7 @@ pub struct SharedSideEffectsPublisher {
struct CandidateGroup {
candidate: ArtifactCandidate,
pinned_envelope_digest: Option<String>,
snapshots: Vec<(PackageKey, String)>,
}
@@ -47,25 +49,34 @@ pub(crate) async fn apply_shared_side_effects(
allow_build_policy: &AllowBuildPolicy,
base_cas_paths: &BaseCasPaths,
side_effects_maps_by_snapshot: &mut SideEffectsMapsBySnapshot,
) {
) -> Vec<ArtifactPinRecord> {
if config.ignore_scripts || config.frozen_store {
return;
return Vec::new();
}
let (Some(server), Some(settings)) =
(config.pnpr_server.as_deref(), config.remote_side_effects_cache.as_ref())
else {
return;
return Vec::new();
};
let Some(platform) = linux_glibc_platform(snapshots) else { return };
let Some(platform) = linux_glibc_platform(snapshots) else { return Vec::new() };
let supported_tags = match linux_glibc_supported_tags(platform) {
Ok(tags) => tags,
Err(error) => {
tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform is unsupported");
return;
return Vec::new();
}
};
let Some(trusted_keys) = decoded_trusted_keys(settings) else { return Vec::new() };
let Some(organization) = non_empty(&settings.organization) else { return Vec::new() };
let owner = OwnerScope::organization(organization.to_string());
let owner_namespace = owner.namespace();
let fingerprint = match platform_fingerprint(&supported_tags) {
Ok(fingerprint) => fingerprint,
Err(error) => {
tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform fingerprint is invalid");
return Vec::new();
}
};
let Some(trusted_keys) = decoded_trusted_keys(settings) else { return };
let Some(organization) = non_empty(&settings.organization) else { return };
let eligible_packages: HashSet<String> = settings.packages.iter().cloned().collect();
let roots: Vec<PackageKey> = in_lockfile_order(snapshots)
@@ -84,7 +95,7 @@ pub(crate) async fn apply_shared_side_effects(
"planned remote side-effects candidates",
);
if roots.is_empty() {
return;
return Vec::new();
}
let graph = build_deps_subgraph(snapshots, packages, roots.clone());
let mut deps_state_cache = pnpm_graph_hasher::DepsStateCache::new();
@@ -137,8 +148,15 @@ pub(crate) async fn apply_shared_side_effects(
version: package_version(&metadata_key, metadata.version.as_deref()),
},
source_integrity,
owner: OwnerScope::organization(organization.to_string()),
owner: owner.clone(),
};
let pinned_envelope_digest = snapshots
.get(&snapshot_key)
.and_then(|snapshot| snapshot.artifact_pins.as_ref())
.and_then(|pins| pins.get(&input_key))
.and_then(|owners| owners.get(&owner_namespace))
.and_then(|platforms| platforms.get(&fingerprint))
.cloned();
if let Some(group) = groups.get_mut(&input_key) {
if group.candidate.package != candidate.package
|| group.candidate.source_integrity != candidate.source_integrity
@@ -147,16 +165,30 @@ pub(crate) async fn apply_shared_side_effects(
collisions.insert(input_key);
continue;
}
if group.pinned_envelope_digest.is_some()
&& pinned_envelope_digest.is_some()
&& group.pinned_envelope_digest != pinned_envelope_digest
{
groups.remove(&input_key);
collisions.insert(input_key);
continue;
}
group.pinned_envelope_digest =
group.pinned_envelope_digest.take().or(pinned_envelope_digest);
group.snapshots.push((snapshot_key, local_cache_key));
} else {
groups.insert(
input_key,
CandidateGroup { candidate, snapshots: vec![(snapshot_key, local_cache_key)] },
CandidateGroup {
candidate,
pinned_envelope_digest,
snapshots: vec![(snapshot_key, local_cache_key)],
},
);
}
}
if groups.is_empty() {
return;
return Vec::new();
}
tracing::debug!(
target: "pacquet::install",
@@ -167,11 +199,17 @@ pub(crate) async fn apply_shared_side_effects(
let client = PnprClient::new(server);
if let Err(error) = client.handshake_artifacts().await {
tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache handshake failed");
return;
return Vec::new();
}
let authorization = config.auth_headers.for_url(server);
let allowed_builds =
groups.values().map(|group| group.candidate.package.name.clone()).collect();
let pinned_envelope_digests = groups
.iter()
.filter_map(|(input_key, group)| {
group.pinned_envelope_digest.as_ref().map(|digest| (input_key.clone(), digest.clone()))
})
.collect();
let resolved = match client
.resolve_artifacts(ResolveArtifactsOptions {
candidates: groups.values().map(|group| group.candidate.clone()).collect(),
@@ -180,6 +218,7 @@ pub(crate) async fn apply_shared_side_effects(
allowed_builds,
ignore_scripts: false,
trusted_keys,
pinned_envelope_digests,
authorization: authorization.clone(),
})
.await
@@ -187,10 +226,20 @@ pub(crate) async fn apply_shared_side_effects(
Ok(resolved) => resolved,
Err(error) => {
tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache lookup failed");
return;
return Vec::new();
}
};
for (input_key, group) in &groups {
if group.pinned_envelope_digest.is_some() && !resolved.contains_key(input_key) {
tracing::warn!(
target: "pacquet::install",
package = %group.candidate.package.name,
"pinned remote side-effects artifact is unavailable; building locally",
);
}
}
let mut artifact_pin_records = Vec::new();
for (input_key, artifact) in resolved {
let Some(group) = groups.get(&input_key) else { continue };
let Some((first_snapshot, _)) = group.snapshots.first() else { continue };
@@ -274,8 +323,16 @@ pub(crate) async fn apply_shared_side_effects(
.map_or_else(HashMap::new, |maps| (**maps).clone());
maps.insert(local_cache_key.clone(), overlay.clone());
side_effects_maps_by_snapshot.insert(snapshot_key.clone(), Arc::new(maps));
artifact_pin_records.push(ArtifactPinRecord {
snapshot_key: snapshot_key.clone(),
input_key: input_key.clone(),
owner: owner_namespace.clone(),
platform_fingerprint: fingerprint.clone(),
envelope_digest: artifact.envelope_digest.clone(),
});
}
}
artifact_pin_records
}
/// Decode the configured trust root, or `None` when it is absent or unusable.
+34 -1
View File
@@ -1,6 +1,6 @@
use crate::{PkgName, SnapshotDepRef};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::collections::{BTreeMap, HashMap};
#[cfg(test)]
mod tests;
@@ -27,6 +27,9 @@ pub struct SnapshotEntry {
)]
pub optional_dependencies: Option<HashMap<PkgName, SnapshotDepRef>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub artifact_pins: Option<ArtifactPins>,
#[serde(skip_serializing_if = "Option::is_none")]
pub transitive_peer_dependencies: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -45,6 +48,36 @@ pub struct SnapshotEntry {
pub optional: bool,
}
pub type ArtifactPins = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
impl SnapshotEntry {
pub fn record_artifact_pin(
&mut self,
input_key: String,
owner: String,
platform_fingerprint: String,
envelope_digest: String,
) -> bool {
let previous = self
.artifact_pins
.as_ref()
.and_then(|pins| pins.get(&input_key))
.and_then(|owners| owners.get(&owner))
.and_then(|platforms| platforms.get(&platform_fingerprint));
if previous == Some(&envelope_digest) {
return false;
}
let pins = self.artifact_pins.get_or_insert_default();
let owners = pins.entry(input_key).or_default();
owners.entry(owner).or_default().insert(platform_fingerprint, envelope_digest);
true
}
pub fn clear_artifact_pins(&mut self) -> bool {
self.artifact_pins.take().is_some()
}
}
#[expect(
clippy::trivially_copy_pass_by_ref,
reason = "serde skip_serializing_if is called as f(&field)"
@@ -34,3 +34,32 @@ fn optional_defaults_false_and_omits_when_false() {
"the `optional` key must not be serialized when false:\n{out}",
);
}
#[test]
fn artifact_pins_round_trip_and_preserve_other_inputs() {
let yaml = text_block! {
"artifactPins:"
" dependency-side-effects:v1:deps=old:"
" organization:acme:"
" linux-node22: abc123"
};
let mut entry: SnapshotEntry = serde_saphyr::from_str(yaml).expect("parse");
assert!(!entry.record_artifact_pin(
"dependency-side-effects:v1:deps=old".to_string(),
"organization:acme".to_string(),
"linux-node22".to_string(),
"abc123".to_string(),
));
assert!(entry.record_artifact_pin(
"dependency-side-effects:v1:deps=new".to_string(),
"organization:acme".to_string(),
"linux-node22".to_string(),
"def456".to_string(),
));
let out = serialize_yaml::to_string(&entry).expect("serialize");
assert!(out.contains("dependency-side-effects:v1:deps=old:"));
assert!(out.contains("dependency-side-effects:v1:deps=new:"));
assert!(out.contains("linux-node22: def456"));
assert!(entry.clear_artifact_pins());
assert!(!entry.clear_artifact_pins());
}
+2 -1
View File
@@ -37,7 +37,7 @@ const INDENT: usize = 2;
/// Per-package / per-snapshot key priority, matching pnpm's lockfile
/// key sort.
const ORDERED_KEYS: [&str; 20] = [
const ORDERED_KEYS: [&str; 21] = [
"resolution",
"id",
"name",
@@ -55,6 +55,7 @@ const ORDERED_KEYS: [&str; 20] = [
"peerDependenciesMeta",
"dependencies",
"optionalDependencies",
"artifactPins",
"transitivePeerDependencies",
"dev",
"optional",
@@ -122,6 +122,7 @@ pub fn build_package_snapshot(
id: None,
dependencies: (!dependencies.is_empty()).then_some(dependencies),
optional_dependencies: None,
artifact_pins: None,
transitive_peer_dependencies: None,
patched: None,
optional: false,
@@ -975,6 +975,7 @@ fn build_snapshot_entry(
id: None,
dependencies: (!dependencies.is_empty()).then_some(dependencies),
optional_dependencies: (!optional_dependencies.is_empty()).then_some(optional_dependencies),
artifact_pins: None,
transitive_peer_dependencies: (!transitive.is_empty()).then_some(transitive),
patched: None,
optional,
@@ -881,6 +881,7 @@ struct InstallRunOptions<'install, 'selection> {
/// whose resolution belongs to a project other than the one that
/// owns that lockfile, so the run must leave it untouched.
save_lockfile: bool,
record_artifact_pins: bool,
/// pnpm's `lockfileCheck`: the caller restores the lockfile and diffs
/// it once the install returns, so the run must leave nothing else on
/// disk changed either. Only `pacquet dedupe --check` sets it.
@@ -903,6 +904,7 @@ impl Default for InstallRunOptions<'_, '_> {
lockfile_specifier_project_manifests: None,
read_package_hooked_manifest_paths: HashSet::new(),
save_lockfile: true,
record_artifact_pins: false,
lockfile_check: false,
manifest_spec_bumps: None,
prompt_eligibility_override: None,
@@ -1041,6 +1043,21 @@ where
.await
}
pub async fn run_with_artifact_pin_recording<Reporter: self::Reporter + 'static>(
self,
selection: Option<WorkspaceInstallSelection<'_>>,
lockfile_verification_override: Option<LockfileVerificationOverride<'a>>,
record_artifact_pins: bool,
) -> Result<(), InstallError> {
Box::pin(self.run_inner::<Reporter>(InstallRunOptions {
lockfile_verification_override,
selection,
record_artifact_pins,
..Default::default()
}))
.await
}
/// Execute the install a legacy `pacquet deploy` runs in its target
/// directory: the deployed manifest is the root importer, while
/// workspace discovery stays anchored at the source workspace so
@@ -273,6 +273,7 @@ struct CommitModulesStateInputs<'a> {
lockfile_was_fast_updated: bool,
save_lockfile: bool,
loaded_wanted_lockfile: Option<&'a Lockfile>,
fresh_wanted_lockfile: Option<&'a Lockfile>,
}
fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), InstallError> {
@@ -300,6 +301,7 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst
lockfile_was_fast_updated,
save_lockfile,
loaded_wanted_lockfile,
fresh_wanted_lockfile,
} = inputs;
let now = SystemTime::now();
let effective_virtual_store_dir = config.effective_virtual_store_dir();
@@ -464,19 +466,20 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst
.map_err(InstallError::SaveCurrentLockfile)?;
}
// Regenerate `pnpm-lock.yaml` from the synthesized snapshot when
// the wanted lockfile was reconstructed from
// `<virtual_store_dir>/lock.yaml`. The no-op short-circuit above
// handles the common case; this branch covers the rare path where
// `.modules.yaml` was wiped or inconsistent and the frozen install
// had to relink.
if take_frozen_path
&& (lockfile_synthesized_from_current
// Persist wanted-lockfile changes discovered during frozen
// materialization. These can be verified artifact pins, or a wanted
// lockfile reconstructed or fast-updated before materialization.
let frozen_wanted_lockfile_to_save = fresh_wanted_lockfile.or_else(|| {
(lockfile_synthesized_from_current
|| lockfile_was_fast_updated
|| config.merge_git_branch_lockfiles)
.then_some(loaded_wanted_lockfile)
.flatten()
});
if take_frozen_path
&& config.lockfile
&& save_lockfile
&& let Some(updated) = loaded_wanted_lockfile
&& let Some(updated) = frozen_wanted_lockfile_to_save
{
updated
.save_to_path(&workspace_root.join(config.wanted_lockfile_name()))
@@ -867,6 +870,7 @@ pub(super) async fn apply_materialization_result<Reporter: self::Reporter + 'sta
lockfile_was_fast_updated,
save_lockfile,
loaded_wanted_lockfile: lockfile,
fresh_wanted_lockfile: fresh_lockfile.as_ref(),
})?;
run_materialized_project_scripts::<Reporter>(MaterializedProjectScriptsInputs {
@@ -60,6 +60,7 @@ pub(super) struct MaterializationInputs<'a, 'install> {
pub(super) pnpmfile_hook: Option<Arc<dyn pnpm_hooks::PnpmfileHooks>>,
pub(super) deploy_manifest_hook: bool,
pub(super) save_lockfile: bool,
pub(super) record_artifact_pins: bool,
pub(super) manifest_spec_bumps: Option<&'a crate::ManifestSpecBumps>,
pub(super) catalogs: &'a Catalogs,
pub(super) prefix: &'a str,
@@ -135,6 +136,7 @@ pub(super) async fn materialize<Reporter: self::Reporter + 'static>(
pnpmfile_hook,
deploy_manifest_hook,
save_lockfile,
record_artifact_pins,
manifest_spec_bumps,
catalogs,
prefix,
@@ -300,11 +302,30 @@ pub(super) async fn materialize<Reporter: self::Reporter + 'static>(
ignored_builds = frozen_result.ignored_builds;
deferred_builds = frozen_result.deferred_builds;
injected_deps = frozen_result.injected_deps;
let fresh_lockfile =
(record_artifact_pins && !frozen_result.artifact_pin_records.is_empty()).then(|| {
let mut updated = lockfile.clone();
for record in frozen_result.artifact_pin_records {
if let Some(snapshot) = updated
.snapshots
.as_mut()
.and_then(|snapshots| snapshots.get_mut(&record.snapshot_key))
{
snapshot.record_artifact_pin(
record.input_key,
record.owner,
record.platform_fingerprint,
record.envelope_digest,
);
}
}
updated
});
(
frozen_result.hoisted_dependencies,
frozen_result.hoisted_locations,
frozen_result.skipped,
None,
fresh_lockfile,
frozen_result.store_index_teardown,
)
} else {
@@ -72,6 +72,7 @@ where
lockfile_specifier_project_manifests,
read_package_hooked_manifest_paths,
save_lockfile,
record_artifact_pins,
lockfile_check: _,
manifest_spec_bumps,
prompt_eligibility_override,
@@ -111,6 +112,8 @@ where
pnpmfile_hook_override,
workspace_projects_override,
} = self;
let record_artifact_pins =
config.lockfile && save_lockfile && (record_artifact_pins || !frozen_lockfile);
http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
http_client_arc.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
let can_prompt = prompt_eligibility_override
@@ -1129,6 +1132,7 @@ where
pnpmfile_hook,
deploy_manifest_hook,
save_lockfile,
record_artifact_pins,
manifest_spec_bumps,
catalogs: &catalogs,
prefix: &prefix,
@@ -1287,7 +1287,7 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
// ~3 ms on the alotta-files fixture, and it is what gets saved
// below anyway.
let phase_start = std::time::Instant::now();
let built_lockfile = build_lockfile(FreshLockfileBuildOptions {
let mut built_lockfile = build_lockfile(FreshLockfileBuildOptions {
config,
importer_manifests: &importer_manifests,
lockfile_specifier_manifests: lockfile_specifier_manifests.as_ref(),
@@ -1475,6 +1475,7 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
// the hoisted-linker pass below to materialize the on-disk
// tree. `None` for the isolated linker.
cas_paths_by_pkg_id,
artifact_pin_records,
} = CreateVirtualStore {
http_client,
config,
@@ -1697,6 +1698,23 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
is_hoisted.then_some(&hoisted_locations),
);
if config.lockfile && save_lockfile {
for record in artifact_pin_records {
if let Some(snapshot) = built_lockfile
.snapshots
.as_mut()
.and_then(|snapshots| snapshots.get_mut(&record.snapshot_key))
{
snapshot.record_artifact_pin(
record.input_key,
record.owner,
record.platform_fingerprint,
record.envelope_digest,
);
}
}
}
// Saved after the build phase succeeds so a partial install can't
// leave a lockfile pointing at slots that never landed on disk.
let (wanted_lockfile, can_record_lockfile_verification) = if !config.lockfile {
@@ -2180,7 +2198,7 @@ fn build_fresh_lockfile(
config.registries_by_prefix.iter().map(|(name, url)| (name.clone(), url.clone())),
)
.collect();
let lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions {
let mut lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions {
importers,
graph,
registry_options_by_url: &config.registry_options_by_url,
@@ -2206,6 +2224,15 @@ fn build_fresh_lockfile(
update_reuse_scopes_by_importer,
time: merge_recorded_time(wanted_lockfile, resolved_time),
})?;
if let (Some(previous), Some(snapshots)) = (
wanted_lockfile.and_then(|lockfile| lockfile.snapshots.as_ref()),
lockfile.snapshots.as_mut(),
) {
for (key, snapshot) in snapshots {
snapshot.artifact_pins =
previous.get(key).and_then(|previous| previous.artifact_pins.clone());
}
}
Ok(lockfile)
}
+8
View File
@@ -336,6 +336,7 @@ pub struct ResolveArtifactsOptions {
pub ignore_scripts: bool,
/// P-256 `SubjectPublicKeyInfo` DER bytes keyed by the envelope's key id.
pub trusted_keys: BTreeMap<String, Vec<u8>>,
pub pinned_envelope_digests: BTreeMap<String, String>,
pub authorization: Option<String>,
}
@@ -577,6 +578,13 @@ impl PnprClient {
.envelope
.digest()
.map_err(|err| PnprClientError::Protocol(err.to_string()))?;
if opts
.pinned_envelope_digests
.get(candidate.key.as_str())
.is_some_and(|pinned| pinned != &envelope_digest)
{
continue;
}
if best.as_ref().is_none_or(|(best_rank, best_digest, _)| {
(rank, &envelope_digest) < (*best_rank, best_digest)
}) {
+41 -1
View File
@@ -896,6 +896,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
allowed_builds: HashSet::from([candidate.package.name.clone()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), untrusted_public_key)]),
pinned_envelope_digests: BTreeMap::new(),
authorization: Some(pnpr_auth.clone()),
})
.await
@@ -912,6 +913,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
allowed_builds: HashSet::from([candidate.package.name.clone()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
pinned_envelope_digests: BTreeMap::new(),
authorization: Some(pnpr_auth.clone()),
})
.await
@@ -925,7 +927,8 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
eligible_packages: HashSet::from([candidate.package.name.clone()]),
allowed_builds: HashSet::from([candidate.package.name.clone()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
pinned_envelope_digests: BTreeMap::new(),
authorization: Some(pnpr_auth.clone()),
})
.await
@@ -933,6 +936,41 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
let artifact = selected.get(&publish.key).expect("trusted compatible variant selected");
assert_eq!(artifact.payload.owner, OwnerScope::organization("pnpr-client"));
assert_eq!(artifact.envelope_digest.len(), 64);
let pinned_digest = artifact.envelope_digest.clone();
let pinned = client
.resolve_artifacts(ResolveArtifactsOptions {
candidates: vec![candidate.clone()],
supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()],
eligible_packages: HashSet::from([candidate.package.name.clone()]),
allowed_builds: HashSet::from([candidate.package.name.clone()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
pinned_envelope_digests: BTreeMap::from([(
candidate.key.clone(),
pinned_digest.clone(),
)]),
authorization: Some(pnpr_auth.clone()),
})
.await
.expect("an exact pin should select the artifact");
assert_eq!(
pinned.get(&candidate.key).map(|artifact| &artifact.envelope_digest),
Some(&pinned_digest),
);
let pinned = client
.resolve_artifacts(ResolveArtifactsOptions {
candidates: vec![candidate.clone()],
supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()],
eligible_packages: HashSet::from([candidate.package.name.clone()]),
allowed_builds: HashSet::from([candidate.package.name.clone()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
pinned_envelope_digests: BTreeMap::from([(candidate.key.clone(), "0".repeat(64))]),
authorization: Some(pnpr_auth.clone()),
})
.await
.expect("a missing pin is a cache miss");
assert!(pinned.is_empty());
let bytes = client
.download_artifact_blob(
@@ -977,6 +1015,7 @@ async fn artifact_lookup_preserves_script_eligibility_and_allow_build_policy() {
allowed_builds,
ignore_scripts,
trusted_keys: trusted_keys.clone(),
pinned_envelope_digests: BTreeMap::new(),
authorization: None,
})
.await
@@ -1091,6 +1130,7 @@ async fn organization_artifact_existence_is_not_exposed_to_another_owner() {
allowed_builds: HashSet::from(["native-addon".to_string()]),
ignore_scripts: false,
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
pinned_envelope_digests: BTreeMap::new(),
authorization: Some(pnpr_auth),
})
.await
@@ -146,6 +146,7 @@ export type InstallDepsOptions = Pick<Config,
}
allowNew?: boolean
forceFullResolution?: boolean
refreshArtifactPins?: boolean
frozenLockfileIfExists?: boolean
include?: IncludedDependencies
includeDirect?: IncludedDependencies
@@ -182,7 +183,7 @@ export type InstallDepsOptions = Pick<Config,
* subcommand — see `runPacquet.ts`'s `noRuntime` opt.
*/
isInstallCommand?: boolean
} & Partial<Pick<Config, 'dryRun' | 'pnpmHomeDir' | 'strictDepBuilds' | 'useLockfile' | 'useGitBranchLockfile' | 'mergeGitBranchLockfiles'>>
} & Partial<Pick<Config, 'dryRun' | 'frozenLockfile' | 'pnpmHomeDir' | 'preferFrozenLockfile' | 'strictDepBuilds' | 'useLockfile' | 'useGitBranchLockfile' | 'mergeGitBranchLockfiles'>>
export async function installDeps (
opts: InstallDepsOptions,
+37 -5
View File
@@ -85,6 +85,7 @@ export function rcOptionsTypes (): Record<string, unknown> {
export function cliOptionsTypes (): Record<string, unknown> {
return {
...rcOptionsTypes(),
'build-artifacts': Boolean,
changeset: Boolean,
'include-github-actions': Boolean,
interactive: Boolean,
@@ -140,6 +141,10 @@ For options that may be used with `-r`, see "pnpm help recursive"',
description: 'Refresh registry revisions without changing package versions',
name: '--patches',
},
{
description: 'Update remote build artifacts without changing package versions',
name: '--build-artifacts',
},
{
description: 'Update packages only in "dependencies" and "optionalDependencies"',
name: '--prod',
@@ -190,6 +195,7 @@ dependencies is not found inside the workspace',
}
export type UpdateCommandOptions = InstallCommandOptions & {
buildArtifacts?: boolean
changeset?: boolean
include?: IncludedDependencies
includeGithubActions?: boolean
@@ -205,6 +211,7 @@ export async function handler (
commands?: CommandHandlerMap
): Promise<string | undefined> {
assertPatchesOptions(params, opts)
assertBuildArtifactsOptions(params, opts)
if (opts.global) {
if (!opts.bin) {
throw new PnpmError('NO_GLOBAL_BIN_DIR', 'Unable to find the global bin directory', {
@@ -418,8 +425,9 @@ async function update (
rebuildHandler?: CommandHandler
): Promise<void> {
assertPatchesOptions(dependencies, opts)
assertBuildArtifactsOptions(dependencies, opts)
const includeDirect = makeIncludeDependenciesFromCLI(opts.cliOptions)
const updateActions = shouldUpdateGitHubActions(opts, includeDirect)
const updateActions = !opts.buildArtifacts && shouldUpdateGitHubActions(opts, includeDirect)
if (opts.latest) {
const dependenciesWithTags = dependencies.filter((name) =>
(!updateActions || !isGitHubActionSelector(name)) && parseUpdateParam(name).versionSpec != null)
@@ -447,7 +455,7 @@ async function update (
} else if ((packageDependencies.length > 0) && depth > 0 && !opts.latest) {
updateMatching = createUpdateMatching(packageDependencies.flatMap(expandUpdateSelectorsForMatching))
}
const generateChangeset = opts.changeset ?? opts.updateConfig?.changeset ?? false
const generateChangeset = !opts.buildArtifacts && (opts.changeset ?? opts.updateConfig?.changeset ?? false)
const changesetContext = generateChangeset ? await captureUpdateChangesetContext(opts) : undefined
if (dependencies.length === 0 || packageDependencies.length > 0) {
await installDeps({
@@ -458,11 +466,12 @@ async function update (
ignoreCurrentSpecifiers: false,
include,
includeDirect,
update: true,
updatePatches: opts.patches,
refreshArtifactPins: opts.buildArtifacts,
update: !opts.buildArtifacts,
updatePatches: opts.buildArtifacts ? false : opts.patches,
updateToLatest: opts.latest,
updateMatching,
updatePackageManifest: opts.patches ? false : opts.save !== false,
updatePackageManifest: opts.buildArtifacts || opts.patches ? false : opts.save !== false,
resolutionMode: opts.save === false ? 'highest' : opts.resolutionMode,
// `--dry-run` is an `install`-only preview; never let a config-level
// `dry-run` turn `update` into a no-op check.
@@ -488,6 +497,29 @@ function assertPatchesOptions (dependencies: string[], opts: UpdateCommandOption
}
}
function assertBuildArtifactsOptions (dependencies: string[], opts: UpdateCommandOptions): void {
if (!opts.buildArtifacts) return
if (dependencies.length > 0 || opts.latest || opts.interactive || opts.global || opts.patches) {
throw new PnpmError('BUILD_ARTIFACTS_WITH_SELECTOR', '--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches')
}
if (opts.useLockfile === false) {
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE',
'Cannot update build artifacts when lockfile is set to false')
}
if (opts.lockfileOnly) {
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY',
'Cannot update build artifacts without downloading and verifying them')
}
if (opts.cliOptions.frozenLockfile === true) {
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE',
'Cannot update build artifacts with a frozen lockfile')
}
opts.frozenLockfile = false
opts.frozenLockfileIfExists = false
opts.preferFrozenLockfile = false
opts.optimisticRepeatInstall = false
}
function shouldUpdateGitHubActions (opts: UpdateCommandOptions, include: IncludedDependencies): boolean {
return include.devDependencies &&
opts.save !== false &&
@@ -3,13 +3,78 @@ import path from 'node:path'
import { beforeAll, describe, expect, it, test } from '@jest/globals'
import type { PnpmError } from '@pnpm/error'
import { install, update } from '@pnpm/installing.commands'
import { readWantedLockfile, writeWantedLockfile } from '@pnpm/lockfile.fs'
import { prepare, preparePackages } from '@pnpm/prepare'
import { addDistTag } from '@pnpm/testing.registry-mock'
import type { ProjectManifest } from '@pnpm/types'
import type { ProjectId, ProjectManifest } from '@pnpm/types'
import { loadJsonFileSync } from 'load-json-file'
import { DEFAULT_OPTS } from '../utils/index.js'
test('update --build-artifacts clears existing pins without updating package versions', async () => {
prepare({ dependencies: { 'is-positive': '1.0.0' } })
await install.handler({ ...DEFAULT_OPTS, dir: process.cwd() })
const lockfile = (await readWantedLockfile('.', { ignoreIncompatible: false }))!
const dependencyResolution = lockfile.importers['.' as ProjectId].dependencies?.['is-positive']
const snapshot = Object.values(lockfile.packages ?? {})[0]
snapshot.artifactPins = {
'dependency-side-effects:v1:deps=old': {
'organization:acme': { platform: '0'.repeat(64) },
},
}
await writeWantedLockfile('.', lockfile)
await update.handler({
...DEFAULT_OPTS,
buildArtifacts: true,
dir: process.cwd(),
})
const refreshed = (await readWantedLockfile('.', { ignoreIncompatible: false }))!
expect(Object.values(refreshed.packages ?? {}).every(pkg => pkg.artifactPins == null)).toBe(true)
expect(refreshed.importers['.' as ProjectId].dependencies?.['is-positive']).toStrictEqual(dependencyResolution)
})
test('update --build-artifacts rejects a frozen lockfile', async () => {
prepare({})
await expect(update.handler({
...DEFAULT_OPTS,
buildArtifacts: true,
cliOptions: { frozenLockfile: true },
dir: process.cwd(),
frozenLockfile: true,
})).rejects.toMatchObject({ code: 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE' })
})
test.each([
['a disabled lockfile', { useLockfile: false }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE'],
['a lockfile-only update', { lockfileOnly: true }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY'],
] as const)('update --build-artifacts rejects %s', async (_description, conflictingOptions, code) => {
prepare({})
await expect(update.handler({
...DEFAULT_OPTS,
...conflictingOptions,
buildArtifacts: true,
dir: process.cwd(),
})).rejects.toMatchObject({ code })
})
test.each([
{ dependencies: ['is-positive'], options: { buildArtifacts: true } },
{ dependencies: [], options: { buildArtifacts: true, latest: true } },
{ dependencies: [], options: { buildArtifacts: true, interactive: true } },
{ dependencies: [], options: { buildArtifacts: true, global: true } },
{ dependencies: [], options: { buildArtifacts: true, patches: true } },
])('update --build-artifacts rejects other update modes', async ({ dependencies, options }) => {
await expect(update.handler({
...DEFAULT_OPTS,
...options,
dir: process.cwd(),
}, dependencies)).rejects.toMatchObject({
code: 'ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR',
})
})
test.each([
{ dependencies: ['is-positive'], options: { patches: true } },
{ dependencies: [], options: { latest: true, patches: true } },
@@ -44,6 +44,7 @@ export interface StrictInstallOptions extends RegistryContext {
forceFullResolution: boolean
fixLockfile: boolean
updateChecksums: boolean
refreshArtifactPins: boolean
dedupe: boolean
ignoreCompatibilityDb: boolean
ignorePackageManifest: boolean
@@ -320,6 +321,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
lockfileDir: opts.lockfileDir ?? opts.dir ?? process.cwd(),
lockfileOnly: false,
updateChecksums: false,
refreshArtifactPins: false,
nodeVersion: opts.nodeVersion,
nodeLinker: 'isolated',
nodeExperimentalPackageMap: false,
@@ -449,6 +451,18 @@ export function extendOptions (
`Cannot generate a ${WANTED_LOCKFILE} because lockfile is set to false`)
}
}
if (extendedOpts.refreshArtifactPins) {
if (!extendedOpts.useLockfile) {
throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_NO_LOCKFILE',
'Cannot refresh artifact pins when lockfile is set to false')
}
if (extendedOpts.lockfileOnly) {
throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_LOCKFILE_ONLY',
'Cannot refresh artifact pins without downloading and verifying artifacts')
}
extendedOpts.force = true
extendedOpts.preferFrozenLockfile = false
}
if (extendedOpts.frozenStore && extendedOpts.force) {
throw new PnpmError('CONFIG_CONFLICT_FROZEN_STORE_WITH_FORCE',
'Cannot use force together with frozenStore: --force re-imports packages into the store, which is opened read-only when frozenStore is enabled')
@@ -369,7 +369,7 @@ export async function mutateModules (
// (pnpm remove), and complete-project revision refreshes. Mutations that
// need other client-side update behavior still fall through to the normal
// flow.
if (opts.pnprServer && canUsePnprForMutations(projects, opts)) {
if (opts.pnprServer && !opts.refreshArtifactPins && canUsePnprForMutations(projects, opts)) {
const pnprResult = await mutateModulesViaPnpr(projects, opts)
if (pnprResult) {
// This path materializes packages of its own, so it verifies the
@@ -404,6 +404,9 @@ export async function mutateModules (
await safeReadProjectManifestOnly(opts.lockfileDir)
let ctx = await getContext(opts)
if (opts.refreshArtifactPins && clearArtifactPins(ctx.wantedLockfile)) {
ctx.wantedLockfileIsModified = true
}
if (!opts.lockfileOnly && !isCheckOnlyInstall(opts) && ctx.modulesFile != null) {
const { purged } = await validateModules(ctx.modulesFile, Object.values(ctx.projects), {
@@ -1502,6 +1505,7 @@ Note that in CI environments, this setting is enabled by default.`,
relinkChangedDependenciesOnly: didFastUpdateOverrides,
wantedLockfile: maybeOpts.ignorePackageManifest ? undefined : ctx.wantedLockfile,
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !frozenLockfile,
verifyLockfile,
})
if (
@@ -2146,6 +2150,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
virtualStoreDir: ctx.virtualStoreDir,
virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
wantedLockfile: newLockfile,
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
wantedToBeSkippedPackageIds,
hoistWorkspacePackages: opts.hoistWorkspacePackages,
virtualStoreOnly: opts.virtualStoreOnly,
@@ -2617,6 +2622,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
prunedAt: ctx.modulesFile?.prunedAt,
wantedLockfile: result.newLockfile,
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
hoistWorkspacePackages: opts.hoistWorkspacePackages,
}))
return {
@@ -2647,6 +2653,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
prunedAt: ctx.modulesFile?.prunedAt,
wantedLockfile: result.newLockfile,
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
hoistWorkspacePackages: opts.hoistWorkspacePackages,
}))
return {
@@ -2740,6 +2747,16 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
const limitLinking = pLimit(16)
function clearArtifactPins (lockfile: LockfileObject): boolean {
let changed = false
for (const snapshot of Object.values(lockfile.packages ?? {})) {
if (snapshot.artifactPins == null) continue
delete snapshot.artifactPins
changed = true
}
return changed
}
async function linkAllBins (
depNodes: DependenciesGraphNode[],
depGraph: DependenciesGraph,
@@ -3142,6 +3159,8 @@ async function installViaPnprServer (
patchedDependencies,
resolvedPatchedDependencies
)
const frozenLockfile = opts.frozenLockfile === true ||
(opts.frozenLockfileIfExists === true && existingLockfile != null)
logger.info({ message: 'Resolving dependencies via the pnpr server', prefix: rootDir })
@@ -3193,7 +3212,7 @@ async function installViaPnprServer (
// Lockfile reuse. Without these the server always reuse-and-updates,
// so `--frozen-lockfile` would silently resolve and rewrite the very
// lockfile it promises to leave alone.
frozenLockfile: opts.frozenLockfile === true || (opts.frozenLockfileIfExists === true && existingLockfile != null),
frozenLockfile,
preferFrozenLockfile: opts.preferFrozenLockfile,
updatePatches: opts.updatePatches,
lockfile: existingLockfile ?? undefined,
@@ -3268,6 +3287,7 @@ async function installViaPnprServer (
patchedDependencies: patchGroups,
skipped: new Set<DepPath>(),
wantedLockfile: lockfile,
recordArtifactPins: opts.useLockfile !== false && opts.saveLockfile !== false && !frozenLockfile,
}
const { ignoredBuilds, stats } = await materializeOrDelegate(
opts,
@@ -72,6 +72,7 @@ export interface LinkPackagesOptions {
sideEffectsCacheRead: boolean
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
pnprServer?: string
recordArtifactPins: boolean
configByUri: Record<string, RegistryConfig>
symlink: boolean
skipped: Set<DepPath>
@@ -171,6 +172,8 @@ export async function linkPackages (projects: ImporterToUpdate[], depGraph: Depe
sideEffectsCacheRead: opts.sideEffectsCacheRead,
remoteSideEffectsCache: opts.remoteSideEffectsCache,
pnprServer: opts.pnprServer,
artifactPinsLockfile: opts.wantedLockfile,
recordArtifactPins: opts.recordArtifactPins,
configByUri: opts.configByUri,
symlink: opts.symlink,
skipped: opts.skipped,
@@ -346,6 +349,8 @@ interface LinkNewPackagesOptions {
sideEffectsCacheRead: boolean
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
pnprServer?: string
artifactPinsLockfile: LockfileObject
recordArtifactPins: boolean
configByUri: Record<string, RegistryConfig>
symlink: boolean
skipped: Set<DepPath>
@@ -466,6 +471,8 @@ async function linkNewPackages (
sideEffectsCacheRead: opts.sideEffectsCacheRead,
remoteSideEffectsCache: opts.remoteSideEffectsCache,
pnprServer: opts.pnprServer,
artifactPinsLockfile: opts.artifactPinsLockfile,
recordArtifactPins: opts.recordArtifactPins,
configByUri: opts.configByUri,
supportedArchitectures: opts.supportedArchitectures,
}),
@@ -526,6 +533,8 @@ async function linkAllPkgs (
sideEffectsCacheRead: boolean
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
pnprServer?: string
artifactPinsLockfile: LockfileObject
recordArtifactPins: boolean
configByUri: Record<string, RegistryConfig>
supportedArchitectures?: SupportedArchitectures
}
@@ -537,12 +546,14 @@ async function linkAllPkgs (
const nodeVersion = findRuntimeNodeVersion(Object.keys(opts.depGraph))
const restorer = createRemoteSideEffectsRestorer({
allowBuild: opts.allowBuild,
artifactPinsLockfile: opts.artifactPinsLockfile,
configByUri: opts.configByUri,
depsGraph: opts.depGraph,
depsStateCache: opts.depsStateCache,
ignoreScripts: opts.ignoreScripts,
nodeVersion,
pnprServer: opts.pnprServer,
recordArtifactPins: opts.recordArtifactPins,
settings: opts.remoteSideEffectsCache,
sideEffectsCacheRead: opts.sideEffectsCacheRead,
storeController,
@@ -109,6 +109,9 @@ function toLockfileDependency (
const result = {
resolution: lockfileResolution,
} as PackageSnapshot
if (opts.prevSnapshot?.artifactPins != null) {
result.artifactPins = opts.prevSnapshot.artifactPins
}
// A registry-qualified dep path (`<name>@<registryName>:<version>`) already
// carries a parseable semver, so the explicit version field written for
// other `:`-containing dep paths would be redundant.
+19 -1
View File
@@ -111,6 +111,7 @@ export interface Project {
}
export interface HeadlessOptions extends RegistryContext {
recordArtifactPins?: boolean
projectDependencies?: Map<ProjectRootDir, ProjectRootDir[]>
allowBuilds?: Record<string, boolean | string>
autoInstallPeers?: boolean
@@ -237,6 +238,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
if (wantedLockfile == null) {
throw new Error(`Headless installation requires a ${WANTED_LOCKFILE} file`)
}
let artifactPinsChanged = false
const depsStateCache: DepsStateCache = {}
// `modulesDir` is conventionally a path relative to `lockfileDir`, but
@@ -448,6 +450,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
if (!skipPostImportLinking) {
await linkHoistedModules(opts.storeController, graph, prevGraph, hierarchy, {
allowBuild,
artifactPinsLockfile: wantedLockfile,
depsStateCache,
disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
force: opts.force,
@@ -457,6 +460,10 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
sideEffectsCacheRead: opts.sideEffectsCacheRead,
remoteSideEffectsCache: opts.remoteSideEffectsCache,
pnprServer: opts.pnprServer,
recordArtifactPins: opts.recordArtifactPins,
onArtifactPinsChanged: () => {
artifactPinsChanged = true
},
configByUri: opts.configByUri,
supportedArchitectures: opts.supportedArchitectures,
})
@@ -490,6 +497,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
}),
linkAllPkgs(opts.storeController, depNodes, {
allowBuild,
artifactPinsLockfile: wantedLockfile,
force: opts.force,
disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
depGraph: graph,
@@ -500,6 +508,10 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
sideEffectsCacheRead: opts.sideEffectsCacheRead,
remoteSideEffectsCache: opts.remoteSideEffectsCache,
pnprServer: opts.pnprServer,
recordArtifactPins: opts.recordArtifactPins,
onArtifactPinsChanged: () => {
artifactPinsChanged = true
},
configByUri: opts.configByUri,
storeDir: opts.storeDir,
supportedArchitectures: opts.supportedArchitectures,
@@ -788,7 +800,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
virtualStoreOnly: opts.virtualStoreOnly,
})
const currentLockfileDir = path.join(rootModulesDir, '.pnpm')
if (opts.useLockfile) {
if (opts.useLockfile || artifactPinsChanged) {
// We need to write the wanted lockfile as well.
// Even though it will only be changed if the workspace will have new projects with no dependencies.
await writeLockfiles({
@@ -1023,6 +1035,7 @@ async function linkAllPkgs (
depNodes: DependenciesGraphNode[],
opts: {
allowBuild?: AllowBuild
artifactPinsLockfile?: LockfileObject
depGraph: DependenciesGraph
depsStateCache: DepsStateCache
disableRelinkLocalDirDeps?: boolean
@@ -1033,6 +1046,8 @@ async function linkAllPkgs (
sideEffectsCacheRead: boolean
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
pnprServer?: string
recordArtifactPins?: boolean
onArtifactPinsChanged?: () => void
configByUri: Record<string, RegistryConfig>
storeDir: string
supportedArchitectures?: SupportedArchitectures
@@ -1056,16 +1071,19 @@ async function linkAllPkgs (
const nodeVersion = findRuntimeNodeVersion(depNodes.map((node) => node.depPath))
const restorer = createRemoteSideEffectsRestorer({
allowBuild: opts.allowBuild,
artifactPinsLockfile: opts.artifactPinsLockfile,
configByUri: opts.configByUri,
depsGraph: opts.depGraph,
depsStateCache: opts.depsStateCache,
ignoreScripts: opts.ignoreScripts,
nodeVersion,
pnprServer: opts.pnprServer,
recordArtifactPins: opts.recordArtifactPins,
settings: opts.remoteSideEffectsCache,
sideEffectsCacheRead: opts.sideEffectsCacheRead,
storeController,
supportedArchitectures: opts.supportedArchitectures,
onArtifactPinsChanged: opts.onArtifactPinsChanged,
warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }),
})
await Promise.all(
@@ -11,6 +11,7 @@ import type {
DepHierarchy,
} from '@pnpm/deps.graph-builder'
import { calcDepState, type DepsStateCache, findRuntimeNodeVersion } from '@pnpm/deps.graph-hasher'
import type { LockfileObject } from '@pnpm/lockfile.fs'
import { logger } from '@pnpm/logger'
import { createRemoteSideEffectsRestorer, type RemoteSideEffectsRestorer } from '@pnpm/pnpr.client'
import type {
@@ -31,6 +32,7 @@ export async function linkHoistedModules (
hierarchy: DepHierarchy,
opts: {
allowBuild?: AllowBuild
artifactPinsLockfile?: LockfileObject
depsStateCache: DepsStateCache
disableRelinkLocalDirDeps?: boolean
force: boolean
@@ -40,6 +42,8 @@ export async function linkHoistedModules (
sideEffectsCacheRead: boolean
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
pnprServer?: string
recordArtifactPins?: boolean
onArtifactPinsChanged?: () => void
configByUri: Record<string, RegistryConfig>
supportedArchitectures?: SupportedArchitectures
}
@@ -67,16 +71,19 @@ export async function linkHoistedModules (
)
const restorer = createRemoteSideEffectsRestorer({
allowBuild: opts.allowBuild,
artifactPinsLockfile: opts.artifactPinsLockfile,
configByUri: opts.configByUri,
depsGraph: graph,
depsStateCache: opts.depsStateCache,
ignoreScripts: opts.ignoreScripts,
nodeVersion,
pnprServer: opts.pnprServer,
recordArtifactPins: opts.recordArtifactPins,
settings: opts.remoteSideEffectsCache,
sideEffectsCacheRead: opts.sideEffectsCacheRead,
storeController,
supportedArchitectures: opts.supportedArchitectures,
onArtifactPinsChanged: opts.onArtifactPinsChanged,
warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }),
})
await Promise.all(
@@ -21,6 +21,7 @@ export function convertToLockfileFile (lockfile: LockfileObject): LockfileFile {
const snapshots: Record<string, LockfilePackageSnapshot> = {}
for (const [depPath, pkg] of Object.entries(lockfile.packages ?? {})) {
snapshots[depPath] = pick([
'artifactPins',
'dependencies',
'optionalDependencies',
'transitivePeerDependencies',
+4 -3
View File
@@ -25,9 +25,10 @@ const ORDERED_KEYS = {
dependencies: 16,
optionalDependencies: 17,
transitivePeerDependencies: 18,
dev: 19,
optional: 20,
artifactPins: 18,
transitivePeerDependencies: 19,
dev: 20,
optional: 21,
}
type RootKey = keyof LockfileFile
@@ -2,7 +2,31 @@ import { expect, test } from '@jest/globals'
import { LOCKFILE_VERSION } from '@pnpm/constants'
import type { DepPath, ProjectId } from '@pnpm/types'
import { convertToLockfileFile } from '../lib/lockfileFormatConverters.js'
import { convertToLockfileFile, convertToLockfileObject } from '../lib/lockfileFormatConverters.js'
test('artifact pins round-trip in package snapshots', () => {
const lockfile = {
lockfileVersion: LOCKFILE_VERSION,
importers: {},
packages: {
['native-addon@1.0.0' as DepPath]: {
artifactPins: {
'dependency-side-effects:v1:deps=abc': {
'organization:acme': { platform: 'digest' },
},
},
resolution: { integrity: 'sha512-AAAA' },
},
},
}
const serialized = convertToLockfileFile(lockfile)
expect(serialized.snapshots?.['native-addon@1.0.0'].artifactPins).toEqual(
lockfile.packages['native-addon@1.0.0'].artifactPins
)
expect(convertToLockfileObject(serialized).packages?.['native-addon@1.0.0' as DepPath].artifactPins).toEqual(
lockfile.packages['native-addon@1.0.0'].artifactPins
)
})
test('empty overrides are removed during lockfile normalization', () => {
expect(convertToLockfileFile({
+3
View File
@@ -32,12 +32,15 @@ export interface LockfileObject extends LockfileBase {
}
export interface LockfilePackageSnapshot {
artifactPins?: ArtifactPins
optional?: true
dependencies?: ResolvedDependencies
optionalDependencies?: ResolvedDependencies
transitivePeerDependencies?: string[]
}
export type ArtifactPins = Record<string, Record<string, Record<string, string>>>
export interface LockfilePackageInfo {
id?: string
patched?: true
+77 -11
View File
@@ -3,7 +3,7 @@ import fs from 'node:fs/promises'
import util from 'node:util'
import { calcDepState, calcDepStateInputKey, type DepsGraph, type DepsStateCache } from '@pnpm/deps.graph-hasher'
import type { LockfileResolution } from '@pnpm/lockfile.types'
import type { ArtifactPins, LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
import { createGetAuthHeaderByURI } from '@pnpm/network.auth-header'
import type { PackageFilesResponse, StoreController, UploadPkgToStoreResult } from '@pnpm/store.controller-types'
import type { AllowBuild, DepPath, RegistryConfig, RemoteSideEffectsCacheSettings, SupportedArchitectures } from '@pnpm/types'
@@ -19,9 +19,12 @@ import {
downloadSharedArtifactBlob,
linuxGlibcCompatibilityTag,
linuxGlibcSupportedTags,
ownerNamespace,
platformFingerprint,
pnprSupportsSharedSideEffects,
publishSharedSideEffects,
resolveSharedSideEffects,
type VerifiedArtifact,
} from './sharedSideEffects.js'
export interface RemoteSideEffectsInstallNode<T extends string> {
@@ -36,16 +39,19 @@ export interface RemoteSideEffectsInstallNode<T extends string> {
export interface RemoteSideEffectsRestorerOptions<T extends string> {
allowBuild?: AllowBuild
artifactPinsLockfile?: LockfileObject
configByUri: Record<string, RegistryConfig>
depsGraph: DepsGraph<T>
depsStateCache: DepsStateCache
ignoreScripts: boolean
nodeVersion?: string
pnprServer?: string
recordArtifactPins?: boolean
settings?: RemoteSideEffectsCacheSettings
sideEffectsCacheRead: boolean
storeController: StoreController
supportedArchitectures?: SupportedArchitectures
onArtifactPinsChanged?: () => void
warn?: (message: string) => void
}
@@ -83,6 +89,7 @@ const MAX_LOOKUP_BATCH = 512
interface RestoredArtifact {
added: Map<string, string>
deleted: string[]
envelopeDigest: string
}
interface QueuedLookup {
@@ -111,6 +118,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
if (platform == null || pnprServer == null || settings == null || organization == null) return undefined
const registryUrl = pnprServer
const ownerName = organization
const owner = { type: 'organization', name: ownerName } as const
let supportedTags: string[]
try {
supportedTags = linuxGlibcSupportedTags(platform)
@@ -119,6 +127,10 @@ export function createRemoteSideEffectsRestorer<T extends string> (
return undefined
}
const trustedKeys = settings.trustedKeys ?? {}
const ownerKey = ownerNamespace(owner)
const fingerprint = platformFingerprint(supportedTags)
const pinnedEnvelopeDigests = new Map<string, string>()
const pinCollisions = new Set<string>()
const eligiblePackages = new Set(settings.packages)
const authorization = createGetAuthHeaderByURI(opts.configByUri)(registryUrl)
const artifactLimit = pLimit(4)
@@ -151,6 +163,21 @@ export function createRemoteSideEffectsRestorer<T extends string> (
patchFileHash: node.patchFileHash,
supportedArchitectures: opts.supportedArchitectures,
})
const pinnedEnvelopeDigest = opts.artifactPinsLockfile?.packages?.[node.depPath]
.artifactPins?.[inputKey]?.[ownerKey]?.[fingerprint]
if (pinnedEnvelopeDigest != null) {
const previous = pinnedEnvelopeDigests.get(inputKey)
if (previous == null) {
pinnedEnvelopeDigests.set(inputKey, pinnedEnvelopeDigest)
} else if (previous !== pinnedEnvelopeDigest) {
pinnedEnvelopeDigests.delete(inputKey)
pinCollisions.add(inputKey)
}
}
if (pinCollisions.has(inputKey)) {
opts.warn?.(`Conflicting remote side-effects pins for ${node.name}@${node.version}; building locally`)
return undefined
}
if (collisions.has(inputKey)) return undefined
const identity = `${node.name}\0${node.version}\0${sourceIntegrity}`
const knownIdentity = identityByInputKey.get(inputKey)
@@ -179,14 +206,20 @@ export function createRemoteSideEffectsRestorer<T extends string> (
key: inputKey,
package: { name: node.name, version: node.version },
sourceIntegrity,
owner: { type: 'organization', name: ownerName },
owner,
})
lookups.set(inputKey, lookup)
}
const artifact = await lookup
if (artifact == null) return undefined
if (artifact == null) {
if (pinnedEnvelopeDigests.has(inputKey)) {
opts.warn?.(`Pinned remote side-effects artifact for ${node.name}@${node.version} is unavailable; building locally`)
}
return undefined
}
recordArtifactPin(node.depPath, inputKey, artifact.envelopeDigest)
node.files.sideEffectsMaps ??= new Map()
node.files.sideEffectsMaps.set(localCacheKey, artifact)
node.files.sideEffectsMaps.set(localCacheKey, { added: artifact.added, deleted: artifact.deleted })
return localCacheKey
}
@@ -216,6 +249,13 @@ export function createRemoteSideEffectsRestorer<T extends string> (
}
async function lookupBatch (batch: QueuedLookup[]): Promise<void> {
const eligibleBatch = batch.filter(({ candidate, resolve }) => {
if (!collisions.has(candidate.key) && !pinCollisions.has(candidate.key)) return true
resolve(undefined)
return false
})
if (eligibleBatch.length === 0) return
const batchPinnedEnvelopeDigests = new Map(pinnedEnvelopeDigests)
supported ??= (async () => {
try {
return await pnprSupportsSharedSideEffects({ registryUrl, authorization })
@@ -225,7 +265,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
}
})()
if (!await supported) {
for (const { resolve } of batch) resolve(undefined)
for (const { resolve } of eligibleBatch) resolve(undefined)
return
}
let resolved
@@ -233,21 +273,26 @@ export function createRemoteSideEffectsRestorer<T extends string> (
resolved = await resolveSharedSideEffects({
registryUrl,
authorization,
candidates: batch.map(({ candidate }) => candidate),
candidates: eligibleBatch.map(({ candidate }) => candidate),
supportedTags,
policy: {
ignoreScripts: false,
eligiblePackages,
allowedBuilds: new Set(batch.map(({ candidate }) => candidate.package.name)),
allowedBuilds: new Set(eligibleBatch.map(({ candidate }) => candidate.package.name)),
},
trustedKeys,
pinnedEnvelopeDigests: batchPinnedEnvelopeDigests,
})
} catch (err: unknown) {
opts.warn?.(`Remote side-effects cache lookup failed: ${errorMessage(err)}`)
for (const { resolve } of batch) resolve(undefined)
for (const { resolve } of eligibleBatch) resolve(undefined)
return
}
await Promise.all(batch.map(async ({ candidate, resolve }) => {
await Promise.all(eligibleBatch.map(async ({ candidate, resolve }) => {
if (collisions.has(candidate.key) || pinCollisions.has(candidate.key)) {
resolve(undefined)
return
}
const artifact = resolved.get(candidate.key)
if (artifact == null) {
resolve(undefined)
@@ -258,7 +303,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
}
async function hydrate (
artifact: { payload: ArtifactPayload },
artifact: VerifiedArtifact,
candidate: ArtifactCandidate
): Promise<RestoredArtifact | undefined> {
try {
@@ -295,12 +340,33 @@ export function createRemoteSideEffectsRestorer<T extends string> (
throw err
}
})))
return { added, deleted: artifact.payload.manifest.deleted }
return { added, deleted: artifact.payload.manifest.deleted, envelopeDigest: artifact.envelopeDigest }
} catch (err: unknown) {
opts.warn?.(`Remote side-effects artifact for ${candidate.package.name}@${candidate.package.version} was rejected: ${errorMessage(err)}`)
return undefined
}
}
function recordArtifactPin (depPath: DepPath, inputKey: string, envelopeDigest: string): void {
if (opts.recordArtifactPins !== true) return
const snapshot = opts.artifactPinsLockfile?.packages?.[depPath]
if (snapshot == null) return
const previous = snapshot.artifactPins?.[inputKey]
if (previous?.[ownerKey]?.[fingerprint] === envelopeDigest) return
const artifactPins: ArtifactPins = {
...snapshot.artifactPins,
[inputKey]: {
...previous,
[ownerKey]: {
...previous?.[ownerKey],
[fingerprint]: envelopeDigest,
},
},
}
snapshot.artifactPins = artifactPins
pinnedEnvelopeDigests.set(inputKey, envelopeDigest)
opts.onArtifactPinsChanged?.()
}
}
export interface PublishBuiltSharedSideEffectsOptions<T extends string> {
+9
View File
@@ -130,6 +130,7 @@ export interface ResolveSharedSideEffectsOptions {
}
/** Base64-encoded P-256 SubjectPublicKeyInfo DER, keyed by key id. */
trustedKeys: Record<string, string>
pinnedEnvelopeDigests?: ReadonlyMap<string, string>
}
interface ArtifactVariant {
@@ -266,6 +267,8 @@ export async function resolveSharedSideEffects (
const rank = compatibilityRank(payload.compatibility, opts.supportedTags)
if (rank == null) continue
const digest = signedArtifactEnvelopeDigest(variant.envelope)
const pinnedDigest = opts.pinnedEnvelopeDigests?.get(candidate.key)
if (pinnedDigest != null && digest !== pinnedDigest) continue
if (
best == null ||
rank < best.rank ||
@@ -286,6 +289,12 @@ export async function resolveSharedSideEffects (
return selected
}
export function ownerNamespace (owner: OwnerScope): string {
return owner.type === 'organization'
? `organization:${owner.name}`
: `publisher:${owner.package}`
}
export function verifySignedArtifactEnvelope (
envelope: SignedArtifactEnvelope,
publicKeySpki: string
@@ -7,7 +7,7 @@ import { setTimeout as delay } from 'node:timers/promises'
import { describe, expect, test } from '@jest/globals'
import type { DepsGraph } from '@pnpm/deps.graph-hasher'
import type { LockfileResolution } from '@pnpm/lockfile.types'
import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
import {
type ArtifactPayload,
createRemoteSideEffectsRestorer,
@@ -38,6 +38,7 @@ describe('install remote side-effects', () => {
'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'),
}
const requestedPaths: string[] = []
let heldResolve: { wait: Promise<void>, notifyStarted: () => void } | undefined
const server = createServer((request, response) => {
const chunks: Buffer[] = []
request.on('data', chunk => chunks.push(Buffer.from(chunk)))
@@ -96,8 +97,18 @@ describe('install remote side-effects', () => {
}],
}
})
response.writeHead(200, { 'content-type': 'application/json' })
.end(JSON.stringify({ artifacts: envelopes }))
const sendResponse = (): void => {
response.writeHead(200, { 'content-type': 'application/json' })
.end(JSON.stringify({ artifacts: envelopes }))
}
const held = heldResolve
if (held == null) {
sendResponse()
} else {
heldResolve = undefined
held.notifyStarted()
void held.wait.then(sendResponse)
}
return
}
if (request.url === '/-/pnpr/v0/artifacts/blob') {
@@ -132,15 +143,32 @@ describe('install remote side-effects', () => {
fullPkgId: graphKey,
},
}
const artifactPinsLockfile: LockfileObject = {
lockfileVersion: '9.0',
importers: {},
packages: {
[depPath]: {
artifactPins: {
'dependency-side-effects:v1:deps=other': {
'organization:acme': { 'other-platform': '1'.repeat(64) },
},
},
resolution: { integrity: sourceIntegrity },
},
},
}
let artifactPinsChanged = 0
try {
const restorer = createRemoteSideEffectsRestorer({
allowBuild: candidate => candidate === depPath,
artifactPinsLockfile,
configByUri: {},
depsGraph,
depsStateCache: {},
ignoreScripts: false,
pnprServer,
recordArtifactPins: true,
settings: {
organization: 'acme',
packages: [packageName],
@@ -148,6 +176,9 @@ describe('install remote side-effects', () => {
},
sideEffectsCacheRead: false,
storeController,
onArtifactPinsChanged: () => {
artifactPinsChanged++
},
})
const cacheKey = await restorer?.restore({
graphKey,
@@ -167,6 +198,17 @@ describe('install remote side-effects', () => {
deleted: ['src/intermediate.o'],
})
expect(storedFiles).toEqual([{ bytes: builtFile, mode: 0o755 }])
const recordedPins = artifactPinsLockfile.packages?.[depPath].artifactPins
expect(Object.keys(recordedPins ?? {})).toHaveLength(2)
expect(recordedPins?.['dependency-side-effects:v1:deps=other'])
.toEqual({ 'organization:acme': { 'other-platform': '1'.repeat(64) } })
const [inputKey, owners] = Object.entries(recordedPins ?? {})
.find(([key]) => key !== 'dependency-side-effects:v1:deps=other')!
const ownerPins = owners['organization:acme']
expect(Object.keys(ownerPins ?? {})).toHaveLength(1)
expect(Object.keys(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/)
expect(Object.values(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/)
expect(artifactPinsChanged).toBe(1)
expect(requestedPaths).toEqual([
'/-/pnpr',
'/-/pnpr/v0/artifacts/resolve',
@@ -210,6 +252,125 @@ describe('install remote side-effects', () => {
expect(storedFiles).toEqual([])
expect(requestedPaths).not.toContain('/-/pnpr/v0/artifacts/blob')
const platformFingerprint = Object.keys(owners['organization:acme'])[0]
const unrelatedPinLockfile: LockfileObject = {
lockfileVersion: '9.0',
importers: {},
packages: {
[depPath]: { resolution: { integrity: sourceIntegrity } },
['stale-package@1.0.0' as DepPath]: {
artifactPins: {
[inputKey]: {
'organization:acme': { [platformFingerprint]: '0'.repeat(64) },
},
},
resolution: { integrity: sourceIntegrity },
},
},
}
const snapshotScoped = createRemoteSideEffectsRestorer({
allowBuild: candidate => candidate === depPath,
artifactPinsLockfile: unrelatedPinLockfile,
configByUri: {},
depsGraph,
depsStateCache: {},
ignoreScripts: false,
pnprServer,
settings: { organization: 'acme', packages: [packageName], trustedKeys },
sideEffectsCacheRead: false,
storeController,
})
const snapshotScopedFiles: PackageFilesResponse = {
filesMap: new Map(),
requiresBuild: true,
resolvedFrom: 'remote',
}
const snapshotScopedKey = await snapshotScoped?.restore({
graphKey,
depPath,
files: snapshotScopedFiles,
name: packageName,
resolution: { integrity: sourceIntegrity } as LockfileResolution,
version: packageVersion,
})
expect(snapshotScopedKey).toBeDefined()
let releaseResolve!: () => void
const waitForRelease = new Promise<void>((resolve) => {
releaseResolve = resolve
})
let notifyResolveStarted!: () => void
const resolveStarted = new Promise<void>((resolve) => {
notifyResolveStarted = resolve
})
heldResolve = { wait: waitForRelease, notifyStarted: notifyResolveStarted }
const conflictingDepPath = `${depPath}(peer@1.0.0)` as DepPath
const envelopeDigest = owners['organization:acme'][platformFingerprint]
const conflictingPinsLockfile: LockfileObject = {
lockfileVersion: '9.0',
importers: {},
packages: {
[depPath]: {
artifactPins: {
[inputKey]: {
'organization:acme': { [platformFingerprint]: envelopeDigest },
},
},
resolution: { integrity: sourceIntegrity },
},
[conflictingDepPath]: {
artifactPins: {
[inputKey]: {
'organization:acme': { [platformFingerprint]: '0'.repeat(64) },
},
},
resolution: { integrity: sourceIntegrity },
},
},
}
const collisionRestorer = createRemoteSideEffectsRestorer({
allowBuild: candidate => candidate === depPath || candidate === conflictingDepPath,
artifactPinsLockfile: conflictingPinsLockfile,
configByUri: {},
depsGraph,
depsStateCache: {},
ignoreScripts: false,
pnprServer,
settings: { organization: 'acme', packages: [packageName], trustedKeys },
sideEffectsCacheRead: false,
storeController,
})!
const firstCollisionFiles: PackageFilesResponse = {
filesMap: new Map(),
requiresBuild: true,
resolvedFrom: 'remote',
}
const firstCollisionRestore = collisionRestorer.restore({
graphKey,
depPath,
files: firstCollisionFiles,
name: packageName,
resolution: { integrity: sourceIntegrity } as LockfileResolution,
version: packageVersion,
})
await resolveStarted
const secondCollisionFiles: PackageFilesResponse = {
filesMap: new Map(),
requiresBuild: true,
resolvedFrom: 'remote',
}
await expect(collisionRestorer.restore({
graphKey,
depPath: conflictingDepPath,
files: secondCollisionFiles,
name: packageName,
resolution: { integrity: sourceIntegrity } as LockfileResolution,
version: packageVersion,
})).resolves.toBeUndefined()
releaseResolve()
await expect(firstCollisionRestore).resolves.toBeUndefined()
expect(firstCollisionFiles.sideEffectsMaps).toBeUndefined()
// Restoring is per package so one package never waits on another's
// fetch, but packages restored together still share a lookup request.
const secondGraphKey = `${graphKey}-second`
+15 -2
View File
@@ -314,7 +314,7 @@ describe('signed shared artifacts', () => {
},
})
expect(mismatchedPackage).toEqual(new Map())
const selected = await resolveSharedSideEffects({
const resolveOptions = {
registryUrl,
authorization: 'Bearer token',
candidates: [{
@@ -332,13 +332,24 @@ describe('signed shared artifacts', () => {
trustedKeys: {
'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'),
},
})
}
const selected = await resolveSharedSideEffects(resolveOptions)
expect(selected.get(payload().inputKey)?.payload).toEqual(payload())
expect(selected.get(payload().inputKey)?.envelopeDigest).toBe(
[envelope, alternateEnvelope]
.map(signedArtifactEnvelopeDigest)
.sort()[0]
)
const pinnedDigest = signedArtifactEnvelopeDigest(alternateEnvelope)
const pinned = await resolveSharedSideEffects({
...resolveOptions,
pinnedEnvelopeDigests: new Map([[payload().inputKey, pinnedDigest]]),
})
expect(pinned.get(payload().inputKey)?.envelopeDigest).toBe(pinnedDigest)
await expect(resolveSharedSideEffects({
...resolveOptions,
pinnedEnvelopeDigests: new Map([[payload().inputKey, '0'.repeat(64)]]),
})).resolves.toEqual(new Map())
await expect(downloadSharedArtifactBlob({
registryUrl,
authorization: 'Bearer token',
@@ -348,6 +359,8 @@ describe('signed shared artifacts', () => {
'/-/pnpr/v0/artifacts',
'/-/pnpr/v0/artifacts/resolve',
'/-/pnpr/v0/artifacts/resolve',
'/-/pnpr/v0/artifacts/resolve',
'/-/pnpr/v0/artifacts/resolve',
'/-/pnpr/v0/artifacts/blob',
])
expect(requests.every(request => request.authorization === 'Bearer token')).toBe(true)