diff --git a/.changeset/artifact-lockfile-pins.md b/.changeset/artifact-lockfile-pins.md new file mode 100644 index 0000000000..dbf7caeee9 --- /dev/null +++ b/.changeset/artifact-lockfile-pins.md @@ -0,0 +1,13 @@ +--- +"@pnpm/lockfile.types": minor +"@pnpm/lockfile.fs": patch +"@pnpm/pnpr.client": minor +"@pnpm/installing.deps-resolver": patch +"@pnpm/installing.deps-restorer": minor +"@pnpm/installing.deps-installer": minor +"@pnpm/installing.commands": minor +"pnpm": minor +"pacquet": minor +--- + +Remote build artifacts are now pinned in `pnpm-lock.yaml` by input key, owner, and consumer platform. Frozen installs enforce existing pins without changing them. Run `pnpm update --build-artifacts` to explicitly replace the recorded pins ([pnpm/pnpm#13771](https://github.com/pnpm/pnpm/issues/13771)). diff --git a/pnpm/crates/cli/src/cli_args/install.rs b/pnpm/crates/cli/src/cli_args/install.rs index ea6ffc24ad..bfc36da97c 100644 --- a/pnpm/crates/cli/src/cli_args/install.rs +++ b/pnpm/crates/cli/src/cli_args/install.rs @@ -244,6 +244,9 @@ pub struct InstallArgs { #[clap(long = "update-checksums")] pub update_checksums: bool, + #[clap(skip)] + pub(crate) refresh_artifact_pins: bool, + /// Maximum number of concurrent network requests during install. #[clap(long = "network-concurrency")] pub network_concurrency: Option, @@ -327,6 +330,7 @@ impl InstallArgs { trust_lockfile: false, no_trust_lockfile: false, update_checksums: false, + refresh_artifact_pins: false, network_concurrency: None, fetch_timeout: None, fetch_warn_timeout_ms: None, @@ -336,6 +340,16 @@ impl InstallArgs { } } + pub(crate) fn for_build_artifact_update( + supported_architectures: SupportedArchitecturesArgs, + ) -> Self { + let mut args = Self::for_reresolving_install(); + args.supported_architectures = supported_architectures; + args.force = true; + args.refresh_artifact_pins = true; + args + } + /// Run the repeat-install fast path before any of the async install /// machinery exists: when every gate below holds and /// [`install_already_up_to_date`] confirms nothing changed since the @@ -366,6 +380,7 @@ impl InstallArgs { if self.effective_frozen_lockfile(config) || self.lockfile_only || self.force + || self.refresh_artifact_pins || self.verify_deps_before_run_install { return false; @@ -495,6 +510,7 @@ impl InstallArgs { ) -> miette::Result<()> { state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::); let frozen_lockfile = match self.configured_frozen_lockfile(state.config) { + _ if self.refresh_artifact_pins => false, Some(value) => value, None if state.config.ci && !self.lockfile_only @@ -550,6 +566,7 @@ impl InstallArgs { trust_lockfile, no_trust_lockfile, update_checksums, + refresh_artifact_pins, network_concurrency: _, fetch_timeout: _, fetch_warn_timeout_ms: _, @@ -566,7 +583,9 @@ impl InstallArgs { // mutual `overrides_with` collapses both spellings to the // last-specified, so at most one is set and the precedence here // is straightforward. - let prefer_frozen_lockfile = if prefer_frozen_lockfile { + let prefer_frozen_lockfile = if refresh_artifact_pins { + Some(false) + } else if prefer_frozen_lockfile { Some(true) } else if no_prefer_frozen_lockfile { Some(false) @@ -607,7 +626,8 @@ impl InstallArgs { // pnpr fast path: when a `pnprServer` URL is configured, offload // resolution + fetching to it, then link `node_modules` from the // server-produced lockfile via the normal frozen install. - if let Some(pnpr_server) = config.pnpr_server.as_deref() { + if let Some(pnpr_server) = config.pnpr_server.as_deref().filter(|_| !refresh_artifact_pins) + { // The pnpr path resolves and links through the server, so it // can't honor `--dry-run`'s no-write contract. Reject up front, // mirroring pnpm's CONFIG_CONFLICT_DRY_RUN_WITH_PNPR_SERVER. @@ -639,6 +659,28 @@ impl InstallArgs { .await; } + if refresh_artifact_pins && !config.lockfile { + return Err(BuildArtifactsWithoutLockfile.into()); + } + let mut refreshed_lockfile = None; + if refresh_artifact_pins { + refreshed_lockfile = lockfile + .get() + .map_err(|error| miette::Report::new(error).wrap_err("load the lockfile"))? + .cloned(); + if let Some(snapshots) = + refreshed_lockfile.as_mut().and_then(|lockfile| lockfile.snapshots.as_mut()) + { + for snapshot in snapshots.values_mut() { + snapshot.clear_artifact_pins(); + } + } + } + let install_lockfile = if refresh_artifact_pins { + MaybeLazyLockfile::Loaded(refreshed_lockfile.as_ref()) + } else { + MaybeLazyLockfile::Lazy(lockfile) + }; let install = Install { tarball_mem_cache: std::sync::Arc::clone(tarball_mem_cache), http_client, @@ -646,7 +688,7 @@ impl InstallArgs { config, manifest, emit_initial_manifest: true, - lockfile: MaybeLazyLockfile::Lazy(lockfile), + lockfile: install_lockfile, lockfile_path: Some(&lockfile_path), dependency_groups: dependency_options.dependency_groups(config.optional), frozen_lockfile, @@ -773,6 +815,11 @@ struct FrozenStoreIncompatibleWithPnpr; )] struct DryRunIncompatibleWithPnpr; +#[derive(Debug, Display, Error, Diagnostic)] +#[display("Cannot update build artifacts when lockfile is set to false")] +#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE))] +struct BuildArtifactsWithoutLockfile; + fn resolve_project( dir: String, manifest: &pnpm_package_manifest::PackageManifest, @@ -1119,25 +1166,13 @@ async fn install_via_pnpr_inner( workspace_projects_override: None, }; - let result = match (selection, lockfile_verification_override) { - (Some(selection), Some(lockfile_verification_override)) => { - Box::pin(install.run_selected_with_lockfile_verification::( - workspace_install_selection(selection), - lockfile_verification_override, - )) - .await - } - (Some(selection), None) => { - Box::pin(install.run_selected::(workspace_install_selection(selection))) - .await - } - (None, Some(lockfile_verification_override)) => { - install - .run_with_lockfile_verification::(lockfile_verification_override) - .await - } - (None, None) => install.run::().await, - }; + let result = install + .run_with_artifact_pin_recording::( + selection.map(workspace_install_selection), + lockfile_verification_override, + !link.frozen_lockfile, + ) + .await; // On failure the prefetcher is dropped, not shut down: shutdown // waits for every in-flight prefetch download (each task holds a // store-index writer handle), which would hold the fail-fast @@ -1353,13 +1388,14 @@ async fn install_via_pnpr_inner( pnpmfile_hook_override: pnpmfile_hook, workspace_projects_override: None, }; - match selection { - Some(selection) => { - Box::pin(install.run_selected::(workspace_install_selection(selection))).await - } - None => install.run::().await, - } - .wrap_err("linking dependencies resolved via the pnpr server")?; + install + .run_with_artifact_pin_recording::( + selection.map(workspace_install_selection), + None, + !link.frozen_lockfile, + ) + .await + .wrap_err("linking dependencies resolved via the pnpr server")?; // The materialization install has awaited every tarball's mem-cache // slot, so all prefetch downloads have finished and queued their diff --git a/pnpm/crates/cli/src/cli_args/pipelines.rs b/pnpm/crates/cli/src/cli_args/pipelines.rs index da23963230..745abb4cbb 100644 --- a/pnpm/crates/cli/src/cli_args/pipelines.rs +++ b/pnpm/crates/cli/src/cli_args/pipelines.rs @@ -712,7 +712,7 @@ pub(crate) fn apply_install_cli_config(cfg: &mut Config, args: &InstallArgs) { cfg.ignore_scripts = resolve_bool_override(args.ignore_scripts, args.no_ignore_scripts, cfg.ignore_scripts); cfg.ignore_pnpmfile = args.ignore_pnpmfile || cfg.ignore_pnpmfile; - cfg.force = args.force || cfg.force; + cfg.force = args.force || args.refresh_artifact_pins || cfg.force; if let Some(network_concurrency) = args.network_concurrency { cfg.network_concurrency = network_concurrency; } diff --git a/pnpm/crates/cli/src/cli_args/update.rs b/pnpm/crates/cli/src/cli_args/update.rs index 23e4018796..886f84ede7 100644 --- a/pnpm/crates/cli/src/cli_args/update.rs +++ b/pnpm/crates/cli/src/cli_args/update.rs @@ -1,6 +1,7 @@ use crate::{ State, cli_args::{ + install::InstallArgs, lockfile_dir::LockfileDirArg, pipelines::InstallFamilySelection, recursive, @@ -94,6 +95,10 @@ pub struct UpdateArgs { #[clap(long)] pub patches: bool, + /// Update remote build artifacts without changing package versions. + #[clap(long = "build-artifacts")] + pub build_artifacts: bool, + /// Write the resolved version without a range operator when /// rewriting the manifest under `--latest`. #[clap(short = 'E', long = "save-exact")] @@ -177,9 +182,22 @@ enum WorkspaceUpdateError { #[diagnostic(code(ERR_PNPM_PATCHES_WITH_SELECTOR))] struct PatchesWithSelectorError; +#[derive(Debug, Display, Error, Diagnostic)] +#[display( + "--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches" +)] +#[diagnostic(code(ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR))] +struct BuildArtifactsWithSelectorError; + +#[derive(Debug, Display, Error, Diagnostic)] +#[display("Cannot update build artifacts without downloading and verifying them")] +#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY))] +struct BuildArtifactsWithLockfileOnlyError; + impl UpdateArgs { pub(crate) fn apply_cli_config(&self, config: &mut Config) { config.ignore_pnpmfile = self.ignore_pnpmfile || config.ignore_pnpmfile; + config.force = self.build_artifacts || config.force; if let Some(pnpr_server) = self.pnpr_server.clone() { config.pnpr_server = Some(pnpr_server); } @@ -190,7 +208,13 @@ impl UpdateArgs { mut state: State, ) -> miette::Result<()> { self.check_patches_options()?; + self.check_build_artifacts_options()?; state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::); + if self.build_artifacts { + return InstallArgs::for_build_artifact_update(self.supported_architectures.clone()) + .run::(state) + .await; + } let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?; let include_direct = self.dependency_options.include_direct(); let update_actions = self.should_update_github_actions(state.config, &include_direct); @@ -320,7 +344,13 @@ impl UpdateArgs { selection: InstallFamilySelection, ) -> miette::Result<()> { self.check_patches_options()?; + self.check_build_artifacts_options()?; state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::); + if self.build_artifacts { + return InstallArgs::for_build_artifact_update(self.supported_architectures.clone()) + .run_selected::(state, selection) + .await; + } let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?; let include_direct = self.dependency_options.include_direct(); let update_actions = self.should_update_github_actions(state.config, &include_direct); @@ -455,6 +485,7 @@ impl UpdateArgs { config: &'static Config, ) -> miette::Result<()> { self.check_patches_options()?; + self.check_build_artifacts_options()?; self.check_workspace_option(None)?; let selected_hashes: Option> = if self.interactive { match crate::cli_args::update_interactive::select_global_package_groups( @@ -515,6 +546,24 @@ impl UpdateArgs { Ok(()) } + fn check_build_artifacts_options(&self) -> miette::Result<()> { + if !self.build_artifacts { + return Ok(()); + } + if !self.packages.is_empty() + || self.latest + || self.interactive + || self.global + || self.patches + { + return Err(BuildArtifactsWithSelectorError.into()); + } + if self.lockfile_only { + return Err(BuildArtifactsWithLockfileOnlyError.into()); + } + Ok(()) + } + fn can_delegate_patch_refresh( &self, update_actions: bool, diff --git a/pnpm/crates/cli/src/cli_args/update/tests.rs b/pnpm/crates/cli/src/cli_args/update/tests.rs index b3ae22299d..132d471c0d 100644 --- a/pnpm/crates/cli/src/cli_args/update/tests.rs +++ b/pnpm/crates/cli/src/cli_args/update/tests.rs @@ -160,6 +160,44 @@ fn patches_is_a_selectorless_update_mode() { } } +#[test] +fn build_artifacts_is_a_selectorless_update_mode() { + let build_artifacts = update_args(&["--build-artifacts"]); + assert!(build_artifacts.build_artifacts); + build_artifacts.check_build_artifacts_options().expect("standalone --build-artifacts"); + + for args in [ + &["--build-artifacts", "foo"][..], + &["--build-artifacts", "--latest"][..], + &["--build-artifacts", "--interactive"][..], + &["--build-artifacts", "--global"][..], + &["--build-artifacts", "--patches"][..], + ] { + let error = update_args(args) + .check_build_artifacts_options() + .expect_err("--build-artifacts combination must fail"); + assert_eq!( + error.to_string(), + "--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches", + ); + } + + let error = update_args(&["--build-artifacts", "--lockfile-only"]) + .check_build_artifacts_options() + .expect_err("--build-artifacts --lockfile-only must fail"); + assert_eq!( + error.to_string(), + "Cannot update build artifacts without downloading and verifying them", + ); +} + +#[test] +fn build_artifacts_forces_reinstallation() { + let mut config = Config::default(); + update_args(&["--build-artifacts"]).apply_cli_config(&mut config); + assert!(config.force); +} + #[test] fn constrained_patch_refresh_stays_on_the_client() { let all_groups = vec![DependencyGroup::Prod, DependencyGroup::Dev, DependencyGroup::Optional]; diff --git a/pnpm/crates/cli/tests/suite/update.rs b/pnpm/crates/cli/tests/suite/update.rs index 765a904e38..df4576f9a8 100644 --- a/pnpm/crates/cli/tests/suite/update.rs +++ b/pnpm/crates/cli/tests/suite/update.rs @@ -25,6 +25,53 @@ const HAS_PRERELEASE: &str = "@pnpm.e2e/has-prerelease"; /// indirect-dependency update behavior when the direct dep is ignored. const PARENT: &str = "@pnpm.e2e/pkg-with-1-dep"; +#[test] +fn build_artifacts_clears_existing_pins_and_overrides_configured_frozen_lockfile() { + let (root, workspace, npmrc_info) = setup(); + let AddMockedRegistry { mock_instance, .. } = npmrc_info; + write_manifest(&workspace, r#"{ "is-positive": "1.0.0" }"#); + pacquet(&workspace, ["install"]).assert().success(); + + let wanted_path = workspace.join("pnpm-lock.yaml"); + let mut wanted = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace) + .expect("load wanted lockfile") + .expect("wanted lockfile"); + wanted + .snapshots + .as_mut() + .and_then(|snapshots| snapshots.values_mut().next()) + .expect("dependency snapshot") + .record_artifact_pin( + "dependency-side-effects:v1:deps=old".to_string(), + "organization:acme".to_string(), + "platform".to_string(), + "0".repeat(64), + ); + wanted.save_to_path(&wanted_path).expect("save artifact pin"); + + let workspace_yaml_path = workspace.join("pnpm-workspace.yaml"); + let mut workspace_yaml = + fs::read_to_string(&workspace_yaml_path).expect("read workspace settings"); + workspace_yaml.push_str("frozenLockfile: true\n"); + fs::write(workspace_yaml_path, workspace_yaml).expect("write workspace settings"); + + pacquet(&workspace, ["update", "--build-artifacts"]).assert().success(); + + let refreshed = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace) + .expect("load refreshed lockfile") + .expect("refreshed lockfile"); + assert!( + refreshed + .snapshots + .as_ref() + .into_iter() + .flat_map(|snapshots| snapshots.values()) + .all(|snapshot| snapshot.artifact_pins.is_none()), + ); + + drop((root, mock_instance)); +} + /// Spin up a temp workspace with the mocked registry and return the /// pieces a multi-step update test needs. fn setup() -> (TempDir, std::path::PathBuf, AddMockedRegistry) { diff --git a/pnpm/crates/deps-restorer/src/build_graph/tests.rs b/pnpm/crates/deps-restorer/src/build_graph/tests.rs index d37271a2c9..bfb6d0cf2e 100644 --- a/pnpm/crates/deps-restorer/src/build_graph/tests.rs +++ b/pnpm/crates/deps-restorer/src/build_graph/tests.rs @@ -35,6 +35,7 @@ fn snap(deps: &[(&str, &str)]) -> SnapshotEntry { id: None, dependencies: (!map.is_empty()).then_some(map), optional_dependencies: None, + artifact_pins: None, transitive_peer_dependencies: None, patched: None, optional: false, diff --git a/pnpm/crates/deps-restorer/src/create_virtual_store.rs b/pnpm/crates/deps-restorer/src/create_virtual_store.rs index eb9bd93e91..32322a1911 100644 --- a/pnpm/crates/deps-restorer/src/create_virtual_store.rs +++ b/pnpm/crates/deps-restorer/src/create_virtual_store.rs @@ -120,6 +120,16 @@ pub struct CreateVirtualStoreOutput { /// the bridge into the link phase instead). Pacquet decouples /// fetch and walk, so the index is built here at fetch time. pub cas_paths_by_pkg_id: Option, + pub artifact_pin_records: Vec, +} + +#[derive(Debug)] +pub struct ArtifactPinRecord { + pub snapshot_key: PackageKey, + pub input_key: String, + pub owner: String, + pub platform_fingerprint: String, + pub envelope_digest: String, } /// This subroutine generates filesystem layout for the virtual store at `node_modules/.pacquet`. @@ -304,6 +314,7 @@ impl CreateVirtualStore<'_> { materialized_snapshots: Vec::new(), fetch_failed: HashSet::new(), cas_paths_by_pkg_id: is_hoisted.then(CasPathsByPkgId::new), + artifact_pin_records: Vec::new(), }); }; let packages = packages.ok_or(CreateVirtualStoreError::MissingPackagesSection)?; @@ -856,7 +867,7 @@ impl CreateVirtualStore<'_> { } } - crate::shared_side_effects::apply_shared_side_effects( + let artifact_pin_records = crate::shared_side_effects::apply_shared_side_effects( config, snapshots, packages, @@ -919,6 +930,7 @@ impl CreateVirtualStore<'_> { materialized_snapshots, fetch_failed, cas_paths_by_pkg_id, + artifact_pin_records, }) } } diff --git a/pnpm/crates/deps-restorer/src/install_frozen_lockfile.rs b/pnpm/crates/deps-restorer/src/install_frozen_lockfile.rs index 153f410919..87fa6a2610 100644 --- a/pnpm/crates/deps-restorer/src/install_frozen_lockfile.rs +++ b/pnpm/crates/deps-restorer/src/install_frozen_lockfile.rs @@ -1,12 +1,13 @@ use crate::{ - AllowBuildPolicy, BuildModules, BuildModulesError, CreateVirtualStore, CreateVirtualStoreError, - CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies, LinkHoistedModulesError, - LinkHoistedModulesOpts, LinkRootComponentMembersError, LinkVirtualStoreBinsError, - LockfileToHoistedDepGraphOptions, SkippedSnapshots, SymlinkDirectDependencies, - SymlinkDirectDependenciesError, SymlinkPackageError, VersionPolicyError, VirtualStoreLayout, - any_installability_constraint, build_direct_deps_by_importer, direct_dep_names_for_importer, - get_hoisted_dependencies, link_hoisted_modules, link_top_level_bins, - lockfile_to_hoisted_dep_graph, symlink_direct_dependencies::importer_root_dir, + AllowBuildPolicy, ArtifactPinRecord, BuildModules, BuildModulesError, CreateVirtualStore, + CreateVirtualStoreError, CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies, + LinkHoistedModulesError, LinkHoistedModulesOpts, LinkRootComponentMembersError, + LinkVirtualStoreBinsError, LockfileToHoistedDepGraphOptions, SkippedSnapshots, + SymlinkDirectDependencies, SymlinkDirectDependenciesError, SymlinkPackageError, + VersionPolicyError, VirtualStoreLayout, any_installability_constraint, + build_direct_deps_by_importer, direct_dep_names_for_importer, get_hoisted_dependencies, + link_hoisted_modules, link_top_level_bins, lockfile_to_hoisted_dep_graph, + symlink_direct_dependencies::importer_root_dir, }; mod build_phase; @@ -668,6 +669,7 @@ where materialized_snapshots, fetch_failed, cas_paths_by_pkg_id, + artifact_pin_records, } = { let mut verify_fut = std::pin::pin!(verify_fut); let mut create_virtual_store_fut = std::pin::pin!(create_virtual_store_fut); @@ -879,6 +881,7 @@ where ignored_builds, deferred_builds, store_index_teardown: writer_task, + artifact_pin_records, }) } } @@ -889,6 +892,7 @@ where /// land without churning every call site. #[derive(Debug)] pub struct InstallFrozenLockfileOutput { + pub artifact_pin_records: Vec, /// Hoisted-dependencies map produced by the isolated-linker /// hoist pass — empty when both hoist patterns are `None` and /// always empty under `nodeLinker: hoisted` (the hoisted diff --git a/pnpm/crates/deps-restorer/src/shared_side_effects.rs b/pnpm/crates/deps-restorer/src/shared_side_effects.rs index d1c3f09673..d1403787c5 100644 --- a/pnpm/crates/deps-restorer/src/shared_side_effects.rs +++ b/pnpm/crates/deps-restorer/src/shared_side_effects.rs @@ -1,6 +1,7 @@ use crate::{ - AllowBuildPolicy, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot, build_deps_subgraph, - deps_graph::in_lockfile_order, install_frozen_lockfile::find_runtime_node_major, + AllowBuildPolicy, ArtifactPinRecord, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot, + build_deps_subgraph, deps_graph::in_lockfile_order, + install_frozen_lockfile::find_runtime_node_major, }; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use pnpm_config::Config; @@ -10,7 +11,7 @@ use pnpm_pnpr_client::{ ArtifactManifest, ArtifactPayload, BuilderProfile, CompatibilityConstraints, LinuxGlibcPlatform, OwnerScope, PackageIdentity, PnprClient, PublishArtifactRequest, ResolveArtifactsOptions, SignedArtifactEnvelope, blob_id, linux_glibc_supported_tags, - linux_glibc_tag, + linux_glibc_tag, platform_fingerprint, }; use sha2::{Digest as _, Sha512}; use std::{ @@ -36,6 +37,7 @@ pub struct SharedSideEffectsPublisher { struct CandidateGroup { candidate: ArtifactCandidate, + pinned_envelope_digest: Option, snapshots: Vec<(PackageKey, String)>, } @@ -47,25 +49,34 @@ pub(crate) async fn apply_shared_side_effects( allow_build_policy: &AllowBuildPolicy, base_cas_paths: &BaseCasPaths, side_effects_maps_by_snapshot: &mut SideEffectsMapsBySnapshot, -) { +) -> Vec { if config.ignore_scripts || config.frozen_store { - return; + return Vec::new(); } let (Some(server), Some(settings)) = (config.pnpr_server.as_deref(), config.remote_side_effects_cache.as_ref()) else { - return; + return Vec::new(); }; - let Some(platform) = linux_glibc_platform(snapshots) else { return }; + let Some(platform) = linux_glibc_platform(snapshots) else { return Vec::new() }; let supported_tags = match linux_glibc_supported_tags(platform) { Ok(tags) => tags, Err(error) => { tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform is unsupported"); - return; + return Vec::new(); + } + }; + let Some(trusted_keys) = decoded_trusted_keys(settings) else { return Vec::new() }; + let Some(organization) = non_empty(&settings.organization) else { return Vec::new() }; + let owner = OwnerScope::organization(organization.to_string()); + let owner_namespace = owner.namespace(); + let fingerprint = match platform_fingerprint(&supported_tags) { + Ok(fingerprint) => fingerprint, + Err(error) => { + tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform fingerprint is invalid"); + return Vec::new(); } }; - let Some(trusted_keys) = decoded_trusted_keys(settings) else { return }; - let Some(organization) = non_empty(&settings.organization) else { return }; let eligible_packages: HashSet = settings.packages.iter().cloned().collect(); let roots: Vec = in_lockfile_order(snapshots) @@ -84,7 +95,7 @@ pub(crate) async fn apply_shared_side_effects( "planned remote side-effects candidates", ); if roots.is_empty() { - return; + return Vec::new(); } let graph = build_deps_subgraph(snapshots, packages, roots.clone()); let mut deps_state_cache = pnpm_graph_hasher::DepsStateCache::new(); @@ -137,8 +148,15 @@ pub(crate) async fn apply_shared_side_effects( version: package_version(&metadata_key, metadata.version.as_deref()), }, source_integrity, - owner: OwnerScope::organization(organization.to_string()), + owner: owner.clone(), }; + let pinned_envelope_digest = snapshots + .get(&snapshot_key) + .and_then(|snapshot| snapshot.artifact_pins.as_ref()) + .and_then(|pins| pins.get(&input_key)) + .and_then(|owners| owners.get(&owner_namespace)) + .and_then(|platforms| platforms.get(&fingerprint)) + .cloned(); if let Some(group) = groups.get_mut(&input_key) { if group.candidate.package != candidate.package || group.candidate.source_integrity != candidate.source_integrity @@ -147,16 +165,30 @@ pub(crate) async fn apply_shared_side_effects( collisions.insert(input_key); continue; } + if group.pinned_envelope_digest.is_some() + && pinned_envelope_digest.is_some() + && group.pinned_envelope_digest != pinned_envelope_digest + { + groups.remove(&input_key); + collisions.insert(input_key); + continue; + } + group.pinned_envelope_digest = + group.pinned_envelope_digest.take().or(pinned_envelope_digest); group.snapshots.push((snapshot_key, local_cache_key)); } else { groups.insert( input_key, - CandidateGroup { candidate, snapshots: vec![(snapshot_key, local_cache_key)] }, + CandidateGroup { + candidate, + pinned_envelope_digest, + snapshots: vec![(snapshot_key, local_cache_key)], + }, ); } } if groups.is_empty() { - return; + return Vec::new(); } tracing::debug!( target: "pacquet::install", @@ -167,11 +199,17 @@ pub(crate) async fn apply_shared_side_effects( let client = PnprClient::new(server); if let Err(error) = client.handshake_artifacts().await { tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache handshake failed"); - return; + return Vec::new(); } let authorization = config.auth_headers.for_url(server); let allowed_builds = groups.values().map(|group| group.candidate.package.name.clone()).collect(); + let pinned_envelope_digests = groups + .iter() + .filter_map(|(input_key, group)| { + group.pinned_envelope_digest.as_ref().map(|digest| (input_key.clone(), digest.clone())) + }) + .collect(); let resolved = match client .resolve_artifacts(ResolveArtifactsOptions { candidates: groups.values().map(|group| group.candidate.clone()).collect(), @@ -180,6 +218,7 @@ pub(crate) async fn apply_shared_side_effects( allowed_builds, ignore_scripts: false, trusted_keys, + pinned_envelope_digests, authorization: authorization.clone(), }) .await @@ -187,10 +226,20 @@ pub(crate) async fn apply_shared_side_effects( Ok(resolved) => resolved, Err(error) => { tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache lookup failed"); - return; + return Vec::new(); } }; + for (input_key, group) in &groups { + if group.pinned_envelope_digest.is_some() && !resolved.contains_key(input_key) { + tracing::warn!( + target: "pacquet::install", + package = %group.candidate.package.name, + "pinned remote side-effects artifact is unavailable; building locally", + ); + } + } + let mut artifact_pin_records = Vec::new(); for (input_key, artifact) in resolved { let Some(group) = groups.get(&input_key) else { continue }; let Some((first_snapshot, _)) = group.snapshots.first() else { continue }; @@ -274,8 +323,16 @@ pub(crate) async fn apply_shared_side_effects( .map_or_else(HashMap::new, |maps| (**maps).clone()); maps.insert(local_cache_key.clone(), overlay.clone()); side_effects_maps_by_snapshot.insert(snapshot_key.clone(), Arc::new(maps)); + artifact_pin_records.push(ArtifactPinRecord { + snapshot_key: snapshot_key.clone(), + input_key: input_key.clone(), + owner: owner_namespace.clone(), + platform_fingerprint: fingerprint.clone(), + envelope_digest: artifact.envelope_digest.clone(), + }); } } + artifact_pin_records } /// Decode the configured trust root, or `None` when it is absent or unusable. diff --git a/pnpm/crates/lockfile/src/snapshot_entry.rs b/pnpm/crates/lockfile/src/snapshot_entry.rs index e58c18cdf6..dae7ea8618 100644 --- a/pnpm/crates/lockfile/src/snapshot_entry.rs +++ b/pnpm/crates/lockfile/src/snapshot_entry.rs @@ -1,6 +1,6 @@ use crate::{PkgName, SnapshotDepRef}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; #[cfg(test)] mod tests; @@ -27,6 +27,9 @@ pub struct SnapshotEntry { )] pub optional_dependencies: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub artifact_pins: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub transitive_peer_dependencies: Option>, #[serde(skip_serializing_if = "Option::is_none")] @@ -45,6 +48,36 @@ pub struct SnapshotEntry { pub optional: bool, } +pub type ArtifactPins = BTreeMap>>; + +impl SnapshotEntry { + pub fn record_artifact_pin( + &mut self, + input_key: String, + owner: String, + platform_fingerprint: String, + envelope_digest: String, + ) -> bool { + let previous = self + .artifact_pins + .as_ref() + .and_then(|pins| pins.get(&input_key)) + .and_then(|owners| owners.get(&owner)) + .and_then(|platforms| platforms.get(&platform_fingerprint)); + if previous == Some(&envelope_digest) { + return false; + } + let pins = self.artifact_pins.get_or_insert_default(); + let owners = pins.entry(input_key).or_default(); + owners.entry(owner).or_default().insert(platform_fingerprint, envelope_digest); + true + } + + pub fn clear_artifact_pins(&mut self) -> bool { + self.artifact_pins.take().is_some() + } +} + #[expect( clippy::trivially_copy_pass_by_ref, reason = "serde skip_serializing_if is called as f(&field)" diff --git a/pnpm/crates/lockfile/src/snapshot_entry/tests.rs b/pnpm/crates/lockfile/src/snapshot_entry/tests.rs index 170a93f144..a51a74bd3c 100644 --- a/pnpm/crates/lockfile/src/snapshot_entry/tests.rs +++ b/pnpm/crates/lockfile/src/snapshot_entry/tests.rs @@ -34,3 +34,32 @@ fn optional_defaults_false_and_omits_when_false() { "the `optional` key must not be serialized when false:\n{out}", ); } + +#[test] +fn artifact_pins_round_trip_and_preserve_other_inputs() { + let yaml = text_block! { + "artifactPins:" + " dependency-side-effects:v1:deps=old:" + " organization:acme:" + " linux-node22: abc123" + }; + let mut entry: SnapshotEntry = serde_saphyr::from_str(yaml).expect("parse"); + assert!(!entry.record_artifact_pin( + "dependency-side-effects:v1:deps=old".to_string(), + "organization:acme".to_string(), + "linux-node22".to_string(), + "abc123".to_string(), + )); + assert!(entry.record_artifact_pin( + "dependency-side-effects:v1:deps=new".to_string(), + "organization:acme".to_string(), + "linux-node22".to_string(), + "def456".to_string(), + )); + let out = serialize_yaml::to_string(&entry).expect("serialize"); + assert!(out.contains("dependency-side-effects:v1:deps=old:")); + assert!(out.contains("dependency-side-effects:v1:deps=new:")); + assert!(out.contains("linux-node22: def456")); + assert!(entry.clear_artifact_pins()); + assert!(!entry.clear_artifact_pins()); +} diff --git a/pnpm/crates/lockfile/src/yaml_emit.rs b/pnpm/crates/lockfile/src/yaml_emit.rs index d3e10b8dc7..c2459f8096 100644 --- a/pnpm/crates/lockfile/src/yaml_emit.rs +++ b/pnpm/crates/lockfile/src/yaml_emit.rs @@ -37,7 +37,7 @@ const INDENT: usize = 2; /// Per-package / per-snapshot key priority, matching pnpm's lockfile /// key sort. -const ORDERED_KEYS: [&str; 20] = [ +const ORDERED_KEYS: [&str; 21] = [ "resolution", "id", "name", @@ -55,6 +55,7 @@ const ORDERED_KEYS: [&str; 20] = [ "peerDependenciesMeta", "dependencies", "optionalDependencies", + "artifactPins", "transitivePeerDependencies", "dev", "optional", diff --git a/pnpm/crates/package-manager/src/build_snapshot.rs b/pnpm/crates/package-manager/src/build_snapshot.rs index 12710c0a81..0e89b9e244 100644 --- a/pnpm/crates/package-manager/src/build_snapshot.rs +++ b/pnpm/crates/package-manager/src/build_snapshot.rs @@ -122,6 +122,7 @@ pub fn build_package_snapshot( id: None, dependencies: (!dependencies.is_empty()).then_some(dependencies), optional_dependencies: None, + artifact_pins: None, transitive_peer_dependencies: None, patched: None, optional: false, diff --git a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs index 3b7ab3caf2..9f5d570b7b 100644 --- a/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs +++ b/pnpm/crates/package-manager/src/dependencies_graph_to_lockfile.rs @@ -975,6 +975,7 @@ fn build_snapshot_entry( id: None, dependencies: (!dependencies.is_empty()).then_some(dependencies), optional_dependencies: (!optional_dependencies.is_empty()).then_some(optional_dependencies), + artifact_pins: None, transitive_peer_dependencies: (!transitive.is_empty()).then_some(transitive), patched: None, optional, diff --git a/pnpm/crates/package-manager/src/install.rs b/pnpm/crates/package-manager/src/install.rs index c045864263..f783f400bd 100644 --- a/pnpm/crates/package-manager/src/install.rs +++ b/pnpm/crates/package-manager/src/install.rs @@ -881,6 +881,7 @@ struct InstallRunOptions<'install, 'selection> { /// whose resolution belongs to a project other than the one that /// owns that lockfile, so the run must leave it untouched. save_lockfile: bool, + record_artifact_pins: bool, /// pnpm's `lockfileCheck`: the caller restores the lockfile and diffs /// it once the install returns, so the run must leave nothing else on /// disk changed either. Only `pacquet dedupe --check` sets it. @@ -903,6 +904,7 @@ impl Default for InstallRunOptions<'_, '_> { lockfile_specifier_project_manifests: None, read_package_hooked_manifest_paths: HashSet::new(), save_lockfile: true, + record_artifact_pins: false, lockfile_check: false, manifest_spec_bumps: None, prompt_eligibility_override: None, @@ -1041,6 +1043,21 @@ where .await } + pub async fn run_with_artifact_pin_recording( + self, + selection: Option>, + lockfile_verification_override: Option>, + record_artifact_pins: bool, + ) -> Result<(), InstallError> { + Box::pin(self.run_inner::(InstallRunOptions { + lockfile_verification_override, + selection, + record_artifact_pins, + ..Default::default() + })) + .await + } + /// Execute the install a legacy `pacquet deploy` runs in its target /// directory: the deployed manifest is the root importer, while /// workspace discovery stays anchored at the source workspace so diff --git a/pnpm/crates/package-manager/src/install/apply_materialization.rs b/pnpm/crates/package-manager/src/install/apply_materialization.rs index 18103ce51a..b1a23569f1 100644 --- a/pnpm/crates/package-manager/src/install/apply_materialization.rs +++ b/pnpm/crates/package-manager/src/install/apply_materialization.rs @@ -273,6 +273,7 @@ struct CommitModulesStateInputs<'a> { lockfile_was_fast_updated: bool, save_lockfile: bool, loaded_wanted_lockfile: Option<&'a Lockfile>, + fresh_wanted_lockfile: Option<&'a Lockfile>, } fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), InstallError> { @@ -300,6 +301,7 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst lockfile_was_fast_updated, save_lockfile, loaded_wanted_lockfile, + fresh_wanted_lockfile, } = inputs; let now = SystemTime::now(); let effective_virtual_store_dir = config.effective_virtual_store_dir(); @@ -464,19 +466,20 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst .map_err(InstallError::SaveCurrentLockfile)?; } - // Regenerate `pnpm-lock.yaml` from the synthesized snapshot when - // the wanted lockfile was reconstructed from - // `/lock.yaml`. The no-op short-circuit above - // handles the common case; this branch covers the rare path where - // `.modules.yaml` was wiped or inconsistent and the frozen install - // had to relink. - if take_frozen_path - && (lockfile_synthesized_from_current + // Persist wanted-lockfile changes discovered during frozen + // materialization. These can be verified artifact pins, or a wanted + // lockfile reconstructed or fast-updated before materialization. + let frozen_wanted_lockfile_to_save = fresh_wanted_lockfile.or_else(|| { + (lockfile_synthesized_from_current || lockfile_was_fast_updated || config.merge_git_branch_lockfiles) + .then_some(loaded_wanted_lockfile) + .flatten() + }); + if take_frozen_path && config.lockfile && save_lockfile - && let Some(updated) = loaded_wanted_lockfile + && let Some(updated) = frozen_wanted_lockfile_to_save { updated .save_to_path(&workspace_root.join(config.wanted_lockfile_name())) @@ -867,6 +870,7 @@ pub(super) async fn apply_materialization_result(MaterializedProjectScriptsInputs { diff --git a/pnpm/crates/package-manager/src/install/materialize.rs b/pnpm/crates/package-manager/src/install/materialize.rs index a749f665e9..2f93c8fd23 100644 --- a/pnpm/crates/package-manager/src/install/materialize.rs +++ b/pnpm/crates/package-manager/src/install/materialize.rs @@ -60,6 +60,7 @@ pub(super) struct MaterializationInputs<'a, 'install> { pub(super) pnpmfile_hook: Option>, pub(super) deploy_manifest_hook: bool, pub(super) save_lockfile: bool, + pub(super) record_artifact_pins: bool, pub(super) manifest_spec_bumps: Option<&'a crate::ManifestSpecBumps>, pub(super) catalogs: &'a Catalogs, pub(super) prefix: &'a str, @@ -135,6 +136,7 @@ pub(super) async fn materialize( pnpmfile_hook, deploy_manifest_hook, save_lockfile, + record_artifact_pins, manifest_spec_bumps, catalogs, prefix, @@ -300,11 +302,30 @@ pub(super) async fn materialize( ignored_builds = frozen_result.ignored_builds; deferred_builds = frozen_result.deferred_builds; injected_deps = frozen_result.injected_deps; + let fresh_lockfile = + (record_artifact_pins && !frozen_result.artifact_pin_records.is_empty()).then(|| { + let mut updated = lockfile.clone(); + for record in frozen_result.artifact_pin_records { + if let Some(snapshot) = updated + .snapshots + .as_mut() + .and_then(|snapshots| snapshots.get_mut(&record.snapshot_key)) + { + snapshot.record_artifact_pin( + record.input_key, + record.owner, + record.platform_fingerprint, + record.envelope_digest, + ); + } + } + updated + }); ( frozen_result.hoisted_dependencies, frozen_result.hoisted_locations, frozen_result.skipped, - None, + fresh_lockfile, frozen_result.store_index_teardown, ) } else { diff --git a/pnpm/crates/package-manager/src/install/run.rs b/pnpm/crates/package-manager/src/install/run.rs index 753e3cfbcb..90086ddcb8 100644 --- a/pnpm/crates/package-manager/src/install/run.rs +++ b/pnpm/crates/package-manager/src/install/run.rs @@ -72,6 +72,7 @@ where lockfile_specifier_project_manifests, read_package_hooked_manifest_paths, save_lockfile, + record_artifact_pins, lockfile_check: _, manifest_spec_bumps, prompt_eligibility_override, @@ -111,6 +112,8 @@ where pnpmfile_hook_override, workspace_projects_override, } = self; + let record_artifact_pins = + config.lockfile && save_lockfile && (record_artifact_pins || !frozen_lockfile); http_client.set_warning_handler(pnpm_reporter::emit_global_warning::); http_client_arc.set_warning_handler(pnpm_reporter::emit_global_warning::); let can_prompt = prompt_eligibility_override @@ -1129,6 +1132,7 @@ where pnpmfile_hook, deploy_manifest_hook, save_lockfile, + record_artifact_pins, manifest_spec_bumps, catalogs: &catalogs, prefix: &prefix, diff --git a/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs b/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs index 87558a1da2..1ce186afb7 100644 --- a/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs +++ b/pnpm/crates/package-manager/src/install_with_fresh_lockfile.rs @@ -1287,7 +1287,7 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { // ~3 ms on the alotta-files fixture, and it is what gets saved // below anyway. let phase_start = std::time::Instant::now(); - let built_lockfile = build_lockfile(FreshLockfileBuildOptions { + let mut built_lockfile = build_lockfile(FreshLockfileBuildOptions { config, importer_manifests: &importer_manifests, lockfile_specifier_manifests: lockfile_specifier_manifests.as_ref(), @@ -1475,6 +1475,7 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { // the hoisted-linker pass below to materialize the on-disk // tree. `None` for the isolated linker. cas_paths_by_pkg_id, + artifact_pin_records, } = CreateVirtualStore { http_client, config, @@ -1697,6 +1698,23 @@ impl InstallWithFreshLockfile<'_, DependencyGroupList> { is_hoisted.then_some(&hoisted_locations), ); + if config.lockfile && save_lockfile { + for record in artifact_pin_records { + if let Some(snapshot) = built_lockfile + .snapshots + .as_mut() + .and_then(|snapshots| snapshots.get_mut(&record.snapshot_key)) + { + snapshot.record_artifact_pin( + record.input_key, + record.owner, + record.platform_fingerprint, + record.envelope_digest, + ); + } + } + } + // Saved after the build phase succeeds so a partial install can't // leave a lockfile pointing at slots that never landed on disk. let (wanted_lockfile, can_record_lockfile_verification) = if !config.lockfile { @@ -2180,7 +2198,7 @@ fn build_fresh_lockfile( config.registries_by_prefix.iter().map(|(name, url)| (name.clone(), url.clone())), ) .collect(); - let lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions { + let mut lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions { importers, graph, registry_options_by_url: &config.registry_options_by_url, @@ -2206,6 +2224,15 @@ fn build_fresh_lockfile( update_reuse_scopes_by_importer, time: merge_recorded_time(wanted_lockfile, resolved_time), })?; + if let (Some(previous), Some(snapshots)) = ( + wanted_lockfile.and_then(|lockfile| lockfile.snapshots.as_ref()), + lockfile.snapshots.as_mut(), + ) { + for (key, snapshot) in snapshots { + snapshot.artifact_pins = + previous.get(key).and_then(|previous| previous.artifact_pins.clone()); + } + } Ok(lockfile) } diff --git a/pnpm/crates/pnpr-client/src/lib.rs b/pnpm/crates/pnpr-client/src/lib.rs index fd479a6112..2d206f522a 100644 --- a/pnpm/crates/pnpr-client/src/lib.rs +++ b/pnpm/crates/pnpr-client/src/lib.rs @@ -336,6 +336,7 @@ pub struct ResolveArtifactsOptions { pub ignore_scripts: bool, /// P-256 `SubjectPublicKeyInfo` DER bytes keyed by the envelope's key id. pub trusted_keys: BTreeMap>, + pub pinned_envelope_digests: BTreeMap, pub authorization: Option, } @@ -577,6 +578,13 @@ impl PnprClient { .envelope .digest() .map_err(|err| PnprClientError::Protocol(err.to_string()))?; + if opts + .pinned_envelope_digests + .get(candidate.key.as_str()) + .is_some_and(|pinned| pinned != &envelope_digest) + { + continue; + } if best.as_ref().is_none_or(|(best_rank, best_digest, _)| { (rank, &envelope_digest) < (*best_rank, best_digest) }) { diff --git a/pnpm/crates/pnpr-client/tests/integration.rs b/pnpm/crates/pnpr-client/tests/integration.rs index f1ebeda639..913b273334 100644 --- a/pnpm/crates/pnpr-client/tests/integration.rs +++ b/pnpm/crates/pnpr-client/tests/integration.rs @@ -896,6 +896,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() { allowed_builds: HashSet::from([candidate.package.name.clone()]), ignore_scripts: false, trusted_keys: BTreeMap::from([("acme-2026".to_string(), untrusted_public_key)]), + pinned_envelope_digests: BTreeMap::new(), authorization: Some(pnpr_auth.clone()), }) .await @@ -912,6 +913,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() { allowed_builds: HashSet::from([candidate.package.name.clone()]), ignore_scripts: false, trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]), + pinned_envelope_digests: BTreeMap::new(), authorization: Some(pnpr_auth.clone()), }) .await @@ -925,7 +927,8 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() { eligible_packages: HashSet::from([candidate.package.name.clone()]), allowed_builds: HashSet::from([candidate.package.name.clone()]), ignore_scripts: false, - trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]), + trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]), + pinned_envelope_digests: BTreeMap::new(), authorization: Some(pnpr_auth.clone()), }) .await @@ -933,6 +936,41 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() { let artifact = selected.get(&publish.key).expect("trusted compatible variant selected"); assert_eq!(artifact.payload.owner, OwnerScope::organization("pnpr-client")); assert_eq!(artifact.envelope_digest.len(), 64); + let pinned_digest = artifact.envelope_digest.clone(); + let pinned = client + .resolve_artifacts(ResolveArtifactsOptions { + candidates: vec![candidate.clone()], + supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()], + eligible_packages: HashSet::from([candidate.package.name.clone()]), + allowed_builds: HashSet::from([candidate.package.name.clone()]), + ignore_scripts: false, + trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]), + pinned_envelope_digests: BTreeMap::from([( + candidate.key.clone(), + pinned_digest.clone(), + )]), + authorization: Some(pnpr_auth.clone()), + }) + .await + .expect("an exact pin should select the artifact"); + assert_eq!( + pinned.get(&candidate.key).map(|artifact| &artifact.envelope_digest), + Some(&pinned_digest), + ); + let pinned = client + .resolve_artifacts(ResolveArtifactsOptions { + candidates: vec![candidate.clone()], + supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()], + eligible_packages: HashSet::from([candidate.package.name.clone()]), + allowed_builds: HashSet::from([candidate.package.name.clone()]), + ignore_scripts: false, + trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]), + pinned_envelope_digests: BTreeMap::from([(candidate.key.clone(), "0".repeat(64))]), + authorization: Some(pnpr_auth.clone()), + }) + .await + .expect("a missing pin is a cache miss"); + assert!(pinned.is_empty()); let bytes = client .download_artifact_blob( @@ -977,6 +1015,7 @@ async fn artifact_lookup_preserves_script_eligibility_and_allow_build_policy() { allowed_builds, ignore_scripts, trusted_keys: trusted_keys.clone(), + pinned_envelope_digests: BTreeMap::new(), authorization: None, }) .await @@ -1091,6 +1130,7 @@ async fn organization_artifact_existence_is_not_exposed_to_another_owner() { allowed_builds: HashSet::from(["native-addon".to_string()]), ignore_scripts: false, trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]), + pinned_envelope_digests: BTreeMap::new(), authorization: Some(pnpr_auth), }) .await diff --git a/pnpm11/installing/commands/src/installDeps.ts b/pnpm11/installing/commands/src/installDeps.ts index 025d2e0ffc..5fbe8c25c6 100644 --- a/pnpm11/installing/commands/src/installDeps.ts +++ b/pnpm11/installing/commands/src/installDeps.ts @@ -146,6 +146,7 @@ export type InstallDepsOptions = Pick> +} & Partial> export async function installDeps ( opts: InstallDepsOptions, diff --git a/pnpm11/installing/commands/src/update/index.ts b/pnpm11/installing/commands/src/update/index.ts index 1e4d73c791..8e04481a0b 100644 --- a/pnpm11/installing/commands/src/update/index.ts +++ b/pnpm11/installing/commands/src/update/index.ts @@ -85,6 +85,7 @@ export function rcOptionsTypes (): Record { export function cliOptionsTypes (): Record { return { ...rcOptionsTypes(), + 'build-artifacts': Boolean, changeset: Boolean, 'include-github-actions': Boolean, interactive: Boolean, @@ -140,6 +141,10 @@ For options that may be used with `-r`, see "pnpm help recursive"', description: 'Refresh registry revisions without changing package versions', name: '--patches', }, + { + description: 'Update remote build artifacts without changing package versions', + name: '--build-artifacts', + }, { description: 'Update packages only in "dependencies" and "optionalDependencies"', name: '--prod', @@ -190,6 +195,7 @@ dependencies is not found inside the workspace', } export type UpdateCommandOptions = InstallCommandOptions & { + buildArtifacts?: boolean changeset?: boolean include?: IncludedDependencies includeGithubActions?: boolean @@ -205,6 +211,7 @@ export async function handler ( commands?: CommandHandlerMap ): Promise { assertPatchesOptions(params, opts) + assertBuildArtifactsOptions(params, opts) if (opts.global) { if (!opts.bin) { throw new PnpmError('NO_GLOBAL_BIN_DIR', 'Unable to find the global bin directory', { @@ -418,8 +425,9 @@ async function update ( rebuildHandler?: CommandHandler ): Promise { assertPatchesOptions(dependencies, opts) + assertBuildArtifactsOptions(dependencies, opts) const includeDirect = makeIncludeDependenciesFromCLI(opts.cliOptions) - const updateActions = shouldUpdateGitHubActions(opts, includeDirect) + const updateActions = !opts.buildArtifacts && shouldUpdateGitHubActions(opts, includeDirect) if (opts.latest) { const dependenciesWithTags = dependencies.filter((name) => (!updateActions || !isGitHubActionSelector(name)) && parseUpdateParam(name).versionSpec != null) @@ -447,7 +455,7 @@ async function update ( } else if ((packageDependencies.length > 0) && depth > 0 && !opts.latest) { updateMatching = createUpdateMatching(packageDependencies.flatMap(expandUpdateSelectorsForMatching)) } - const generateChangeset = opts.changeset ?? opts.updateConfig?.changeset ?? false + const generateChangeset = !opts.buildArtifacts && (opts.changeset ?? opts.updateConfig?.changeset ?? false) const changesetContext = generateChangeset ? await captureUpdateChangesetContext(opts) : undefined if (dependencies.length === 0 || packageDependencies.length > 0) { await installDeps({ @@ -458,11 +466,12 @@ async function update ( ignoreCurrentSpecifiers: false, include, includeDirect, - update: true, - updatePatches: opts.patches, + refreshArtifactPins: opts.buildArtifacts, + update: !opts.buildArtifacts, + updatePatches: opts.buildArtifacts ? false : opts.patches, updateToLatest: opts.latest, updateMatching, - updatePackageManifest: opts.patches ? false : opts.save !== false, + updatePackageManifest: opts.buildArtifacts || opts.patches ? false : opts.save !== false, resolutionMode: opts.save === false ? 'highest' : opts.resolutionMode, // `--dry-run` is an `install`-only preview; never let a config-level // `dry-run` turn `update` into a no-op check. @@ -488,6 +497,29 @@ function assertPatchesOptions (dependencies: string[], opts: UpdateCommandOption } } +function assertBuildArtifactsOptions (dependencies: string[], opts: UpdateCommandOptions): void { + if (!opts.buildArtifacts) return + if (dependencies.length > 0 || opts.latest || opts.interactive || opts.global || opts.patches) { + throw new PnpmError('BUILD_ARTIFACTS_WITH_SELECTOR', '--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches') + } + if (opts.useLockfile === false) { + throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE', + 'Cannot update build artifacts when lockfile is set to false') + } + if (opts.lockfileOnly) { + throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY', + 'Cannot update build artifacts without downloading and verifying them') + } + if (opts.cliOptions.frozenLockfile === true) { + throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE', + 'Cannot update build artifacts with a frozen lockfile') + } + opts.frozenLockfile = false + opts.frozenLockfileIfExists = false + opts.preferFrozenLockfile = false + opts.optimisticRepeatInstall = false +} + function shouldUpdateGitHubActions (opts: UpdateCommandOptions, include: IncludedDependencies): boolean { return include.devDependencies && opts.save !== false && diff --git a/pnpm11/installing/commands/test/update/update.ts b/pnpm11/installing/commands/test/update/update.ts index bd21be41fa..292217585e 100644 --- a/pnpm11/installing/commands/test/update/update.ts +++ b/pnpm11/installing/commands/test/update/update.ts @@ -3,13 +3,78 @@ import path from 'node:path' import { beforeAll, describe, expect, it, test } from '@jest/globals' import type { PnpmError } from '@pnpm/error' import { install, update } from '@pnpm/installing.commands' +import { readWantedLockfile, writeWantedLockfile } from '@pnpm/lockfile.fs' import { prepare, preparePackages } from '@pnpm/prepare' import { addDistTag } from '@pnpm/testing.registry-mock' -import type { ProjectManifest } from '@pnpm/types' +import type { ProjectId, ProjectManifest } from '@pnpm/types' import { loadJsonFileSync } from 'load-json-file' import { DEFAULT_OPTS } from '../utils/index.js' +test('update --build-artifacts clears existing pins without updating package versions', async () => { + prepare({ dependencies: { 'is-positive': '1.0.0' } }) + await install.handler({ ...DEFAULT_OPTS, dir: process.cwd() }) + const lockfile = (await readWantedLockfile('.', { ignoreIncompatible: false }))! + const dependencyResolution = lockfile.importers['.' as ProjectId].dependencies?.['is-positive'] + const snapshot = Object.values(lockfile.packages ?? {})[0] + snapshot.artifactPins = { + 'dependency-side-effects:v1:deps=old': { + 'organization:acme': { platform: '0'.repeat(64) }, + }, + } + await writeWantedLockfile('.', lockfile) + + await update.handler({ + ...DEFAULT_OPTS, + buildArtifacts: true, + dir: process.cwd(), + }) + + const refreshed = (await readWantedLockfile('.', { ignoreIncompatible: false }))! + expect(Object.values(refreshed.packages ?? {}).every(pkg => pkg.artifactPins == null)).toBe(true) + expect(refreshed.importers['.' as ProjectId].dependencies?.['is-positive']).toStrictEqual(dependencyResolution) +}) + +test('update --build-artifacts rejects a frozen lockfile', async () => { + prepare({}) + await expect(update.handler({ + ...DEFAULT_OPTS, + buildArtifacts: true, + cliOptions: { frozenLockfile: true }, + dir: process.cwd(), + frozenLockfile: true, + })).rejects.toMatchObject({ code: 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE' }) +}) + +test.each([ + ['a disabled lockfile', { useLockfile: false }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE'], + ['a lockfile-only update', { lockfileOnly: true }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY'], +] as const)('update --build-artifacts rejects %s', async (_description, conflictingOptions, code) => { + prepare({}) + await expect(update.handler({ + ...DEFAULT_OPTS, + ...conflictingOptions, + buildArtifacts: true, + dir: process.cwd(), + })).rejects.toMatchObject({ code }) +}) + +test.each([ + { dependencies: ['is-positive'], options: { buildArtifacts: true } }, + { dependencies: [], options: { buildArtifacts: true, latest: true } }, + { dependencies: [], options: { buildArtifacts: true, interactive: true } }, + { dependencies: [], options: { buildArtifacts: true, global: true } }, + { dependencies: [], options: { buildArtifacts: true, patches: true } }, +])('update --build-artifacts rejects other update modes', async ({ dependencies, options }) => { + await expect(update.handler({ + ...DEFAULT_OPTS, + ...options, + dir: process.cwd(), + }, dependencies)).rejects.toMatchObject({ + code: 'ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR', + }) +}) + test.each([ { dependencies: ['is-positive'], options: { patches: true } }, { dependencies: [], options: { latest: true, patches: true } }, diff --git a/pnpm11/installing/deps-installer/src/install/extendInstallOptions.ts b/pnpm11/installing/deps-installer/src/install/extendInstallOptions.ts index d134ae54c5..0f8bd268c0 100644 --- a/pnpm11/installing/deps-installer/src/install/extendInstallOptions.ts +++ b/pnpm11/installing/deps-installer/src/install/extendInstallOptions.ts @@ -44,6 +44,7 @@ export interface StrictInstallOptions extends RegistryContext { forceFullResolution: boolean fixLockfile: boolean updateChecksums: boolean + refreshArtifactPins: boolean dedupe: boolean ignoreCompatibilityDb: boolean ignorePackageManifest: boolean @@ -320,6 +321,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => { lockfileDir: opts.lockfileDir ?? opts.dir ?? process.cwd(), lockfileOnly: false, updateChecksums: false, + refreshArtifactPins: false, nodeVersion: opts.nodeVersion, nodeLinker: 'isolated', nodeExperimentalPackageMap: false, @@ -449,6 +451,18 @@ export function extendOptions ( `Cannot generate a ${WANTED_LOCKFILE} because lockfile is set to false`) } } + if (extendedOpts.refreshArtifactPins) { + if (!extendedOpts.useLockfile) { + throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_NO_LOCKFILE', + 'Cannot refresh artifact pins when lockfile is set to false') + } + if (extendedOpts.lockfileOnly) { + throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_LOCKFILE_ONLY', + 'Cannot refresh artifact pins without downloading and verifying artifacts') + } + extendedOpts.force = true + extendedOpts.preferFrozenLockfile = false + } if (extendedOpts.frozenStore && extendedOpts.force) { throw new PnpmError('CONFIG_CONFLICT_FROZEN_STORE_WITH_FORCE', 'Cannot use force together with frozenStore: --force re-imports packages into the store, which is opened read-only when frozenStore is enabled') diff --git a/pnpm11/installing/deps-installer/src/install/index.ts b/pnpm11/installing/deps-installer/src/install/index.ts index 61c935f5a8..a5c99c34e9 100644 --- a/pnpm11/installing/deps-installer/src/install/index.ts +++ b/pnpm11/installing/deps-installer/src/install/index.ts @@ -369,7 +369,7 @@ export async function mutateModules ( // (pnpm remove), and complete-project revision refreshes. Mutations that // need other client-side update behavior still fall through to the normal // flow. - if (opts.pnprServer && canUsePnprForMutations(projects, opts)) { + if (opts.pnprServer && !opts.refreshArtifactPins && canUsePnprForMutations(projects, opts)) { const pnprResult = await mutateModulesViaPnpr(projects, opts) if (pnprResult) { // This path materializes packages of its own, so it verifies the @@ -404,6 +404,9 @@ export async function mutateModules ( await safeReadProjectManifestOnly(opts.lockfileDir) let ctx = await getContext(opts) + if (opts.refreshArtifactPins && clearArtifactPins(ctx.wantedLockfile)) { + ctx.wantedLockfileIsModified = true + } if (!opts.lockfileOnly && !isCheckOnlyInstall(opts) && ctx.modulesFile != null) { const { purged } = await validateModules(ctx.modulesFile, Object.values(ctx.projects), { @@ -1502,6 +1505,7 @@ Note that in CI environments, this setting is enabled by default.`, relinkChangedDependenciesOnly: didFastUpdateOverrides, wantedLockfile: maybeOpts.ignorePackageManifest ? undefined : ctx.wantedLockfile, useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified, + recordArtifactPins: opts.useLockfile && opts.saveLockfile && !frozenLockfile, verifyLockfile, }) if ( @@ -2146,6 +2150,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => { virtualStoreDir: ctx.virtualStoreDir, virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength, wantedLockfile: newLockfile, + recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile, wantedToBeSkippedPackageIds, hoistWorkspacePackages: opts.hoistWorkspacePackages, virtualStoreOnly: opts.virtualStoreOnly, @@ -2617,6 +2622,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => { prunedAt: ctx.modulesFile?.prunedAt, wantedLockfile: result.newLockfile, useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified, + recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile, hoistWorkspacePackages: opts.hoistWorkspacePackages, })) return { @@ -2647,6 +2653,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => { prunedAt: ctx.modulesFile?.prunedAt, wantedLockfile: result.newLockfile, useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified, + recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile, hoistWorkspacePackages: opts.hoistWorkspacePackages, })) return { @@ -2740,6 +2747,16 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => { const limitLinking = pLimit(16) +function clearArtifactPins (lockfile: LockfileObject): boolean { + let changed = false + for (const snapshot of Object.values(lockfile.packages ?? {})) { + if (snapshot.artifactPins == null) continue + delete snapshot.artifactPins + changed = true + } + return changed +} + async function linkAllBins ( depNodes: DependenciesGraphNode[], depGraph: DependenciesGraph, @@ -3142,6 +3159,8 @@ async function installViaPnprServer ( patchedDependencies, resolvedPatchedDependencies ) + const frozenLockfile = opts.frozenLockfile === true || + (opts.frozenLockfileIfExists === true && existingLockfile != null) logger.info({ message: 'Resolving dependencies via the pnpr server', prefix: rootDir }) @@ -3193,7 +3212,7 @@ async function installViaPnprServer ( // Lockfile reuse. Without these the server always reuse-and-updates, // so `--frozen-lockfile` would silently resolve and rewrite the very // lockfile it promises to leave alone. - frozenLockfile: opts.frozenLockfile === true || (opts.frozenLockfileIfExists === true && existingLockfile != null), + frozenLockfile, preferFrozenLockfile: opts.preferFrozenLockfile, updatePatches: opts.updatePatches, lockfile: existingLockfile ?? undefined, @@ -3268,6 +3287,7 @@ async function installViaPnprServer ( patchedDependencies: patchGroups, skipped: new Set(), wantedLockfile: lockfile, + recordArtifactPins: opts.useLockfile !== false && opts.saveLockfile !== false && !frozenLockfile, } const { ignoredBuilds, stats } = await materializeOrDelegate( opts, diff --git a/pnpm11/installing/deps-installer/src/install/link.ts b/pnpm11/installing/deps-installer/src/install/link.ts index 91334af729..ed5beef244 100644 --- a/pnpm11/installing/deps-installer/src/install/link.ts +++ b/pnpm11/installing/deps-installer/src/install/link.ts @@ -72,6 +72,7 @@ export interface LinkPackagesOptions { sideEffectsCacheRead: boolean remoteSideEffectsCache?: RemoteSideEffectsCacheSettings pnprServer?: string + recordArtifactPins: boolean configByUri: Record symlink: boolean skipped: Set @@ -171,6 +172,8 @@ export async function linkPackages (projects: ImporterToUpdate[], depGraph: Depe sideEffectsCacheRead: opts.sideEffectsCacheRead, remoteSideEffectsCache: opts.remoteSideEffectsCache, pnprServer: opts.pnprServer, + artifactPinsLockfile: opts.wantedLockfile, + recordArtifactPins: opts.recordArtifactPins, configByUri: opts.configByUri, symlink: opts.symlink, skipped: opts.skipped, @@ -346,6 +349,8 @@ interface LinkNewPackagesOptions { sideEffectsCacheRead: boolean remoteSideEffectsCache?: RemoteSideEffectsCacheSettings pnprServer?: string + artifactPinsLockfile: LockfileObject + recordArtifactPins: boolean configByUri: Record symlink: boolean skipped: Set @@ -466,6 +471,8 @@ async function linkNewPackages ( sideEffectsCacheRead: opts.sideEffectsCacheRead, remoteSideEffectsCache: opts.remoteSideEffectsCache, pnprServer: opts.pnprServer, + artifactPinsLockfile: opts.artifactPinsLockfile, + recordArtifactPins: opts.recordArtifactPins, configByUri: opts.configByUri, supportedArchitectures: opts.supportedArchitectures, }), @@ -526,6 +533,8 @@ async function linkAllPkgs ( sideEffectsCacheRead: boolean remoteSideEffectsCache?: RemoteSideEffectsCacheSettings pnprServer?: string + artifactPinsLockfile: LockfileObject + recordArtifactPins: boolean configByUri: Record supportedArchitectures?: SupportedArchitectures } @@ -537,12 +546,14 @@ async function linkAllPkgs ( const nodeVersion = findRuntimeNodeVersion(Object.keys(opts.depGraph)) const restorer = createRemoteSideEffectsRestorer({ allowBuild: opts.allowBuild, + artifactPinsLockfile: opts.artifactPinsLockfile, configByUri: opts.configByUri, depsGraph: opts.depGraph, depsStateCache: opts.depsStateCache, ignoreScripts: opts.ignoreScripts, nodeVersion, pnprServer: opts.pnprServer, + recordArtifactPins: opts.recordArtifactPins, settings: opts.remoteSideEffectsCache, sideEffectsCacheRead: opts.sideEffectsCacheRead, storeController, diff --git a/pnpm11/installing/deps-resolver/src/updateLockfile.ts b/pnpm11/installing/deps-resolver/src/updateLockfile.ts index f1d9b62f97..8cc88b8146 100644 --- a/pnpm11/installing/deps-resolver/src/updateLockfile.ts +++ b/pnpm11/installing/deps-resolver/src/updateLockfile.ts @@ -109,6 +109,9 @@ function toLockfileDependency ( const result = { resolution: lockfileResolution, } as PackageSnapshot + if (opts.prevSnapshot?.artifactPins != null) { + result.artifactPins = opts.prevSnapshot.artifactPins + } // A registry-qualified dep path (`@:`) already // carries a parseable semver, so the explicit version field written for // other `:`-containing dep paths would be redundant. diff --git a/pnpm11/installing/deps-restorer/src/index.ts b/pnpm11/installing/deps-restorer/src/index.ts index e3da7e383d..de62d5ed92 100644 --- a/pnpm11/installing/deps-restorer/src/index.ts +++ b/pnpm11/installing/deps-restorer/src/index.ts @@ -111,6 +111,7 @@ export interface Project { } export interface HeadlessOptions extends RegistryContext { + recordArtifactPins?: boolean projectDependencies?: Map allowBuilds?: Record autoInstallPeers?: boolean @@ -237,6 +238,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise { + artifactPinsChanged = true + }, configByUri: opts.configByUri, supportedArchitectures: opts.supportedArchitectures, }) @@ -490,6 +497,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise { + artifactPinsChanged = true + }, configByUri: opts.configByUri, storeDir: opts.storeDir, supportedArchitectures: opts.supportedArchitectures, @@ -788,7 +800,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise void configByUri: Record storeDir: string supportedArchitectures?: SupportedArchitectures @@ -1056,16 +1071,19 @@ async function linkAllPkgs ( const nodeVersion = findRuntimeNodeVersion(depNodes.map((node) => node.depPath)) const restorer = createRemoteSideEffectsRestorer({ allowBuild: opts.allowBuild, + artifactPinsLockfile: opts.artifactPinsLockfile, configByUri: opts.configByUri, depsGraph: opts.depGraph, depsStateCache: opts.depsStateCache, ignoreScripts: opts.ignoreScripts, nodeVersion, pnprServer: opts.pnprServer, + recordArtifactPins: opts.recordArtifactPins, settings: opts.remoteSideEffectsCache, sideEffectsCacheRead: opts.sideEffectsCacheRead, storeController, supportedArchitectures: opts.supportedArchitectures, + onArtifactPinsChanged: opts.onArtifactPinsChanged, warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }), }) await Promise.all( diff --git a/pnpm11/installing/deps-restorer/src/linkHoistedModules.ts b/pnpm11/installing/deps-restorer/src/linkHoistedModules.ts index ee8a255a26..dec3ac57ee 100644 --- a/pnpm11/installing/deps-restorer/src/linkHoistedModules.ts +++ b/pnpm11/installing/deps-restorer/src/linkHoistedModules.ts @@ -11,6 +11,7 @@ import type { DepHierarchy, } from '@pnpm/deps.graph-builder' import { calcDepState, type DepsStateCache, findRuntimeNodeVersion } from '@pnpm/deps.graph-hasher' +import type { LockfileObject } from '@pnpm/lockfile.fs' import { logger } from '@pnpm/logger' import { createRemoteSideEffectsRestorer, type RemoteSideEffectsRestorer } from '@pnpm/pnpr.client' import type { @@ -31,6 +32,7 @@ export async function linkHoistedModules ( hierarchy: DepHierarchy, opts: { allowBuild?: AllowBuild + artifactPinsLockfile?: LockfileObject depsStateCache: DepsStateCache disableRelinkLocalDirDeps?: boolean force: boolean @@ -40,6 +42,8 @@ export async function linkHoistedModules ( sideEffectsCacheRead: boolean remoteSideEffectsCache?: RemoteSideEffectsCacheSettings pnprServer?: string + recordArtifactPins?: boolean + onArtifactPinsChanged?: () => void configByUri: Record supportedArchitectures?: SupportedArchitectures } @@ -67,16 +71,19 @@ export async function linkHoistedModules ( ) const restorer = createRemoteSideEffectsRestorer({ allowBuild: opts.allowBuild, + artifactPinsLockfile: opts.artifactPinsLockfile, configByUri: opts.configByUri, depsGraph: graph, depsStateCache: opts.depsStateCache, ignoreScripts: opts.ignoreScripts, nodeVersion, pnprServer: opts.pnprServer, + recordArtifactPins: opts.recordArtifactPins, settings: opts.remoteSideEffectsCache, sideEffectsCacheRead: opts.sideEffectsCacheRead, storeController, supportedArchitectures: opts.supportedArchitectures, + onArtifactPinsChanged: opts.onArtifactPinsChanged, warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }), }) await Promise.all( diff --git a/pnpm11/lockfile/fs/src/lockfileFormatConverters.ts b/pnpm11/lockfile/fs/src/lockfileFormatConverters.ts index c069a8a257..32d0fbd3f0 100644 --- a/pnpm11/lockfile/fs/src/lockfileFormatConverters.ts +++ b/pnpm11/lockfile/fs/src/lockfileFormatConverters.ts @@ -21,6 +21,7 @@ export function convertToLockfileFile (lockfile: LockfileObject): LockfileFile { const snapshots: Record = {} for (const [depPath, pkg] of Object.entries(lockfile.packages ?? {})) { snapshots[depPath] = pick([ + 'artifactPins', 'dependencies', 'optionalDependencies', 'transitivePeerDependencies', diff --git a/pnpm11/lockfile/fs/src/sortLockfileKeys.ts b/pnpm11/lockfile/fs/src/sortLockfileKeys.ts index 30044e47e2..8ee0578637 100644 --- a/pnpm11/lockfile/fs/src/sortLockfileKeys.ts +++ b/pnpm11/lockfile/fs/src/sortLockfileKeys.ts @@ -25,9 +25,10 @@ const ORDERED_KEYS = { dependencies: 16, optionalDependencies: 17, - transitivePeerDependencies: 18, - dev: 19, - optional: 20, + artifactPins: 18, + transitivePeerDependencies: 19, + dev: 20, + optional: 21, } type RootKey = keyof LockfileFile diff --git a/pnpm11/lockfile/fs/test/normalizeLockfile.test.ts b/pnpm11/lockfile/fs/test/normalizeLockfile.test.ts index 82ecc53ab6..d6c26e4609 100644 --- a/pnpm11/lockfile/fs/test/normalizeLockfile.test.ts +++ b/pnpm11/lockfile/fs/test/normalizeLockfile.test.ts @@ -2,7 +2,31 @@ import { expect, test } from '@jest/globals' import { LOCKFILE_VERSION } from '@pnpm/constants' import type { DepPath, ProjectId } from '@pnpm/types' -import { convertToLockfileFile } from '../lib/lockfileFormatConverters.js' +import { convertToLockfileFile, convertToLockfileObject } from '../lib/lockfileFormatConverters.js' + +test('artifact pins round-trip in package snapshots', () => { + const lockfile = { + lockfileVersion: LOCKFILE_VERSION, + importers: {}, + packages: { + ['native-addon@1.0.0' as DepPath]: { + artifactPins: { + 'dependency-side-effects:v1:deps=abc': { + 'organization:acme': { platform: 'digest' }, + }, + }, + resolution: { integrity: 'sha512-AAAA' }, + }, + }, + } + const serialized = convertToLockfileFile(lockfile) + expect(serialized.snapshots?.['native-addon@1.0.0'].artifactPins).toEqual( + lockfile.packages['native-addon@1.0.0'].artifactPins + ) + expect(convertToLockfileObject(serialized).packages?.['native-addon@1.0.0' as DepPath].artifactPins).toEqual( + lockfile.packages['native-addon@1.0.0'].artifactPins + ) +}) test('empty overrides are removed during lockfile normalization', () => { expect(convertToLockfileFile({ diff --git a/pnpm11/lockfile/types/src/index.ts b/pnpm11/lockfile/types/src/index.ts index a3148238be..a79042c93c 100644 --- a/pnpm11/lockfile/types/src/index.ts +++ b/pnpm11/lockfile/types/src/index.ts @@ -32,12 +32,15 @@ export interface LockfileObject extends LockfileBase { } export interface LockfilePackageSnapshot { + artifactPins?: ArtifactPins optional?: true dependencies?: ResolvedDependencies optionalDependencies?: ResolvedDependencies transitivePeerDependencies?: string[] } +export type ArtifactPins = Record>> + export interface LockfilePackageInfo { id?: string patched?: true diff --git a/pnpr/client/src/installSharedSideEffects.ts b/pnpr/client/src/installSharedSideEffects.ts index 41201a68b7..21d1ea8f98 100644 --- a/pnpr/client/src/installSharedSideEffects.ts +++ b/pnpr/client/src/installSharedSideEffects.ts @@ -3,7 +3,7 @@ import fs from 'node:fs/promises' import util from 'node:util' import { calcDepState, calcDepStateInputKey, type DepsGraph, type DepsStateCache } from '@pnpm/deps.graph-hasher' -import type { LockfileResolution } from '@pnpm/lockfile.types' +import type { ArtifactPins, LockfileObject, LockfileResolution } from '@pnpm/lockfile.types' import { createGetAuthHeaderByURI } from '@pnpm/network.auth-header' import type { PackageFilesResponse, StoreController, UploadPkgToStoreResult } from '@pnpm/store.controller-types' import type { AllowBuild, DepPath, RegistryConfig, RemoteSideEffectsCacheSettings, SupportedArchitectures } from '@pnpm/types' @@ -19,9 +19,12 @@ import { downloadSharedArtifactBlob, linuxGlibcCompatibilityTag, linuxGlibcSupportedTags, + ownerNamespace, + platformFingerprint, pnprSupportsSharedSideEffects, publishSharedSideEffects, resolveSharedSideEffects, + type VerifiedArtifact, } from './sharedSideEffects.js' export interface RemoteSideEffectsInstallNode { @@ -36,16 +39,19 @@ export interface RemoteSideEffectsInstallNode { export interface RemoteSideEffectsRestorerOptions { allowBuild?: AllowBuild + artifactPinsLockfile?: LockfileObject configByUri: Record depsGraph: DepsGraph depsStateCache: DepsStateCache ignoreScripts: boolean nodeVersion?: string pnprServer?: string + recordArtifactPins?: boolean settings?: RemoteSideEffectsCacheSettings sideEffectsCacheRead: boolean storeController: StoreController supportedArchitectures?: SupportedArchitectures + onArtifactPinsChanged?: () => void warn?: (message: string) => void } @@ -83,6 +89,7 @@ const MAX_LOOKUP_BATCH = 512 interface RestoredArtifact { added: Map deleted: string[] + envelopeDigest: string } interface QueuedLookup { @@ -111,6 +118,7 @@ export function createRemoteSideEffectsRestorer ( if (platform == null || pnprServer == null || settings == null || organization == null) return undefined const registryUrl = pnprServer const ownerName = organization + const owner = { type: 'organization', name: ownerName } as const let supportedTags: string[] try { supportedTags = linuxGlibcSupportedTags(platform) @@ -119,6 +127,10 @@ export function createRemoteSideEffectsRestorer ( return undefined } const trustedKeys = settings.trustedKeys ?? {} + const ownerKey = ownerNamespace(owner) + const fingerprint = platformFingerprint(supportedTags) + const pinnedEnvelopeDigests = new Map() + const pinCollisions = new Set() const eligiblePackages = new Set(settings.packages) const authorization = createGetAuthHeaderByURI(opts.configByUri)(registryUrl) const artifactLimit = pLimit(4) @@ -151,6 +163,21 @@ export function createRemoteSideEffectsRestorer ( patchFileHash: node.patchFileHash, supportedArchitectures: opts.supportedArchitectures, }) + const pinnedEnvelopeDigest = opts.artifactPinsLockfile?.packages?.[node.depPath] + .artifactPins?.[inputKey]?.[ownerKey]?.[fingerprint] + if (pinnedEnvelopeDigest != null) { + const previous = pinnedEnvelopeDigests.get(inputKey) + if (previous == null) { + pinnedEnvelopeDigests.set(inputKey, pinnedEnvelopeDigest) + } else if (previous !== pinnedEnvelopeDigest) { + pinnedEnvelopeDigests.delete(inputKey) + pinCollisions.add(inputKey) + } + } + if (pinCollisions.has(inputKey)) { + opts.warn?.(`Conflicting remote side-effects pins for ${node.name}@${node.version}; building locally`) + return undefined + } if (collisions.has(inputKey)) return undefined const identity = `${node.name}\0${node.version}\0${sourceIntegrity}` const knownIdentity = identityByInputKey.get(inputKey) @@ -179,14 +206,20 @@ export function createRemoteSideEffectsRestorer ( key: inputKey, package: { name: node.name, version: node.version }, sourceIntegrity, - owner: { type: 'organization', name: ownerName }, + owner, }) lookups.set(inputKey, lookup) } const artifact = await lookup - if (artifact == null) return undefined + if (artifact == null) { + if (pinnedEnvelopeDigests.has(inputKey)) { + opts.warn?.(`Pinned remote side-effects artifact for ${node.name}@${node.version} is unavailable; building locally`) + } + return undefined + } + recordArtifactPin(node.depPath, inputKey, artifact.envelopeDigest) node.files.sideEffectsMaps ??= new Map() - node.files.sideEffectsMaps.set(localCacheKey, artifact) + node.files.sideEffectsMaps.set(localCacheKey, { added: artifact.added, deleted: artifact.deleted }) return localCacheKey } @@ -216,6 +249,13 @@ export function createRemoteSideEffectsRestorer ( } async function lookupBatch (batch: QueuedLookup[]): Promise { + const eligibleBatch = batch.filter(({ candidate, resolve }) => { + if (!collisions.has(candidate.key) && !pinCollisions.has(candidate.key)) return true + resolve(undefined) + return false + }) + if (eligibleBatch.length === 0) return + const batchPinnedEnvelopeDigests = new Map(pinnedEnvelopeDigests) supported ??= (async () => { try { return await pnprSupportsSharedSideEffects({ registryUrl, authorization }) @@ -225,7 +265,7 @@ export function createRemoteSideEffectsRestorer ( } })() if (!await supported) { - for (const { resolve } of batch) resolve(undefined) + for (const { resolve } of eligibleBatch) resolve(undefined) return } let resolved @@ -233,21 +273,26 @@ export function createRemoteSideEffectsRestorer ( resolved = await resolveSharedSideEffects({ registryUrl, authorization, - candidates: batch.map(({ candidate }) => candidate), + candidates: eligibleBatch.map(({ candidate }) => candidate), supportedTags, policy: { ignoreScripts: false, eligiblePackages, - allowedBuilds: new Set(batch.map(({ candidate }) => candidate.package.name)), + allowedBuilds: new Set(eligibleBatch.map(({ candidate }) => candidate.package.name)), }, trustedKeys, + pinnedEnvelopeDigests: batchPinnedEnvelopeDigests, }) } catch (err: unknown) { opts.warn?.(`Remote side-effects cache lookup failed: ${errorMessage(err)}`) - for (const { resolve } of batch) resolve(undefined) + for (const { resolve } of eligibleBatch) resolve(undefined) return } - await Promise.all(batch.map(async ({ candidate, resolve }) => { + await Promise.all(eligibleBatch.map(async ({ candidate, resolve }) => { + if (collisions.has(candidate.key) || pinCollisions.has(candidate.key)) { + resolve(undefined) + return + } const artifact = resolved.get(candidate.key) if (artifact == null) { resolve(undefined) @@ -258,7 +303,7 @@ export function createRemoteSideEffectsRestorer ( } async function hydrate ( - artifact: { payload: ArtifactPayload }, + artifact: VerifiedArtifact, candidate: ArtifactCandidate ): Promise { try { @@ -295,12 +340,33 @@ export function createRemoteSideEffectsRestorer ( throw err } }))) - return { added, deleted: artifact.payload.manifest.deleted } + return { added, deleted: artifact.payload.manifest.deleted, envelopeDigest: artifact.envelopeDigest } } catch (err: unknown) { opts.warn?.(`Remote side-effects artifact for ${candidate.package.name}@${candidate.package.version} was rejected: ${errorMessage(err)}`) return undefined } } + + function recordArtifactPin (depPath: DepPath, inputKey: string, envelopeDigest: string): void { + if (opts.recordArtifactPins !== true) return + const snapshot = opts.artifactPinsLockfile?.packages?.[depPath] + if (snapshot == null) return + const previous = snapshot.artifactPins?.[inputKey] + if (previous?.[ownerKey]?.[fingerprint] === envelopeDigest) return + const artifactPins: ArtifactPins = { + ...snapshot.artifactPins, + [inputKey]: { + ...previous, + [ownerKey]: { + ...previous?.[ownerKey], + [fingerprint]: envelopeDigest, + }, + }, + } + snapshot.artifactPins = artifactPins + pinnedEnvelopeDigests.set(inputKey, envelopeDigest) + opts.onArtifactPinsChanged?.() + } } export interface PublishBuiltSharedSideEffectsOptions { diff --git a/pnpr/client/src/sharedSideEffects.ts b/pnpr/client/src/sharedSideEffects.ts index c4c6ac9b2d..f8c67f663f 100644 --- a/pnpr/client/src/sharedSideEffects.ts +++ b/pnpr/client/src/sharedSideEffects.ts @@ -130,6 +130,7 @@ export interface ResolveSharedSideEffectsOptions { } /** Base64-encoded P-256 SubjectPublicKeyInfo DER, keyed by key id. */ trustedKeys: Record + pinnedEnvelopeDigests?: ReadonlyMap } interface ArtifactVariant { @@ -266,6 +267,8 @@ export async function resolveSharedSideEffects ( const rank = compatibilityRank(payload.compatibility, opts.supportedTags) if (rank == null) continue const digest = signedArtifactEnvelopeDigest(variant.envelope) + const pinnedDigest = opts.pinnedEnvelopeDigests?.get(candidate.key) + if (pinnedDigest != null && digest !== pinnedDigest) continue if ( best == null || rank < best.rank || @@ -286,6 +289,12 @@ export async function resolveSharedSideEffects ( return selected } +export function ownerNamespace (owner: OwnerScope): string { + return owner.type === 'organization' + ? `organization:${owner.name}` + : `publisher:${owner.package}` +} + export function verifySignedArtifactEnvelope ( envelope: SignedArtifactEnvelope, publicKeySpki: string diff --git a/pnpr/client/test/installSharedSideEffects.test.ts b/pnpr/client/test/installSharedSideEffects.test.ts index 569d00d838..05093ff5c8 100644 --- a/pnpr/client/test/installSharedSideEffects.test.ts +++ b/pnpr/client/test/installSharedSideEffects.test.ts @@ -7,7 +7,7 @@ import { setTimeout as delay } from 'node:timers/promises' import { describe, expect, test } from '@jest/globals' import type { DepsGraph } from '@pnpm/deps.graph-hasher' -import type { LockfileResolution } from '@pnpm/lockfile.types' +import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types' import { type ArtifactPayload, createRemoteSideEffectsRestorer, @@ -38,6 +38,7 @@ describe('install remote side-effects', () => { 'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'), } const requestedPaths: string[] = [] + let heldResolve: { wait: Promise, notifyStarted: () => void } | undefined const server = createServer((request, response) => { const chunks: Buffer[] = [] request.on('data', chunk => chunks.push(Buffer.from(chunk))) @@ -96,8 +97,18 @@ describe('install remote side-effects', () => { }], } }) - response.writeHead(200, { 'content-type': 'application/json' }) - .end(JSON.stringify({ artifacts: envelopes })) + const sendResponse = (): void => { + response.writeHead(200, { 'content-type': 'application/json' }) + .end(JSON.stringify({ artifacts: envelopes })) + } + const held = heldResolve + if (held == null) { + sendResponse() + } else { + heldResolve = undefined + held.notifyStarted() + void held.wait.then(sendResponse) + } return } if (request.url === '/-/pnpr/v0/artifacts/blob') { @@ -132,15 +143,32 @@ describe('install remote side-effects', () => { fullPkgId: graphKey, }, } + const artifactPinsLockfile: LockfileObject = { + lockfileVersion: '9.0', + importers: {}, + packages: { + [depPath]: { + artifactPins: { + 'dependency-side-effects:v1:deps=other': { + 'organization:acme': { 'other-platform': '1'.repeat(64) }, + }, + }, + resolution: { integrity: sourceIntegrity }, + }, + }, + } + let artifactPinsChanged = 0 try { const restorer = createRemoteSideEffectsRestorer({ allowBuild: candidate => candidate === depPath, + artifactPinsLockfile, configByUri: {}, depsGraph, depsStateCache: {}, ignoreScripts: false, pnprServer, + recordArtifactPins: true, settings: { organization: 'acme', packages: [packageName], @@ -148,6 +176,9 @@ describe('install remote side-effects', () => { }, sideEffectsCacheRead: false, storeController, + onArtifactPinsChanged: () => { + artifactPinsChanged++ + }, }) const cacheKey = await restorer?.restore({ graphKey, @@ -167,6 +198,17 @@ describe('install remote side-effects', () => { deleted: ['src/intermediate.o'], }) expect(storedFiles).toEqual([{ bytes: builtFile, mode: 0o755 }]) + const recordedPins = artifactPinsLockfile.packages?.[depPath].artifactPins + expect(Object.keys(recordedPins ?? {})).toHaveLength(2) + expect(recordedPins?.['dependency-side-effects:v1:deps=other']) + .toEqual({ 'organization:acme': { 'other-platform': '1'.repeat(64) } }) + const [inputKey, owners] = Object.entries(recordedPins ?? {}) + .find(([key]) => key !== 'dependency-side-effects:v1:deps=other')! + const ownerPins = owners['organization:acme'] + expect(Object.keys(ownerPins ?? {})).toHaveLength(1) + expect(Object.keys(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/) + expect(Object.values(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/) + expect(artifactPinsChanged).toBe(1) expect(requestedPaths).toEqual([ '/-/pnpr', '/-/pnpr/v0/artifacts/resolve', @@ -210,6 +252,125 @@ describe('install remote side-effects', () => { expect(storedFiles).toEqual([]) expect(requestedPaths).not.toContain('/-/pnpr/v0/artifacts/blob') + const platformFingerprint = Object.keys(owners['organization:acme'])[0] + const unrelatedPinLockfile: LockfileObject = { + lockfileVersion: '9.0', + importers: {}, + packages: { + [depPath]: { resolution: { integrity: sourceIntegrity } }, + ['stale-package@1.0.0' as DepPath]: { + artifactPins: { + [inputKey]: { + 'organization:acme': { [platformFingerprint]: '0'.repeat(64) }, + }, + }, + resolution: { integrity: sourceIntegrity }, + }, + }, + } + const snapshotScoped = createRemoteSideEffectsRestorer({ + allowBuild: candidate => candidate === depPath, + artifactPinsLockfile: unrelatedPinLockfile, + configByUri: {}, + depsGraph, + depsStateCache: {}, + ignoreScripts: false, + pnprServer, + settings: { organization: 'acme', packages: [packageName], trustedKeys }, + sideEffectsCacheRead: false, + storeController, + }) + const snapshotScopedFiles: PackageFilesResponse = { + filesMap: new Map(), + requiresBuild: true, + resolvedFrom: 'remote', + } + const snapshotScopedKey = await snapshotScoped?.restore({ + graphKey, + depPath, + files: snapshotScopedFiles, + name: packageName, + resolution: { integrity: sourceIntegrity } as LockfileResolution, + version: packageVersion, + }) + expect(snapshotScopedKey).toBeDefined() + + let releaseResolve!: () => void + const waitForRelease = new Promise((resolve) => { + releaseResolve = resolve + }) + let notifyResolveStarted!: () => void + const resolveStarted = new Promise((resolve) => { + notifyResolveStarted = resolve + }) + heldResolve = { wait: waitForRelease, notifyStarted: notifyResolveStarted } + const conflictingDepPath = `${depPath}(peer@1.0.0)` as DepPath + const envelopeDigest = owners['organization:acme'][platformFingerprint] + const conflictingPinsLockfile: LockfileObject = { + lockfileVersion: '9.0', + importers: {}, + packages: { + [depPath]: { + artifactPins: { + [inputKey]: { + 'organization:acme': { [platformFingerprint]: envelopeDigest }, + }, + }, + resolution: { integrity: sourceIntegrity }, + }, + [conflictingDepPath]: { + artifactPins: { + [inputKey]: { + 'organization:acme': { [platformFingerprint]: '0'.repeat(64) }, + }, + }, + resolution: { integrity: sourceIntegrity }, + }, + }, + } + const collisionRestorer = createRemoteSideEffectsRestorer({ + allowBuild: candidate => candidate === depPath || candidate === conflictingDepPath, + artifactPinsLockfile: conflictingPinsLockfile, + configByUri: {}, + depsGraph, + depsStateCache: {}, + ignoreScripts: false, + pnprServer, + settings: { organization: 'acme', packages: [packageName], trustedKeys }, + sideEffectsCacheRead: false, + storeController, + })! + const firstCollisionFiles: PackageFilesResponse = { + filesMap: new Map(), + requiresBuild: true, + resolvedFrom: 'remote', + } + const firstCollisionRestore = collisionRestorer.restore({ + graphKey, + depPath, + files: firstCollisionFiles, + name: packageName, + resolution: { integrity: sourceIntegrity } as LockfileResolution, + version: packageVersion, + }) + await resolveStarted + const secondCollisionFiles: PackageFilesResponse = { + filesMap: new Map(), + requiresBuild: true, + resolvedFrom: 'remote', + } + await expect(collisionRestorer.restore({ + graphKey, + depPath: conflictingDepPath, + files: secondCollisionFiles, + name: packageName, + resolution: { integrity: sourceIntegrity } as LockfileResolution, + version: packageVersion, + })).resolves.toBeUndefined() + releaseResolve() + await expect(firstCollisionRestore).resolves.toBeUndefined() + expect(firstCollisionFiles.sideEffectsMaps).toBeUndefined() + // Restoring is per package so one package never waits on another's // fetch, but packages restored together still share a lookup request. const secondGraphKey = `${graphKey}-second` diff --git a/pnpr/client/test/sharedSideEffects.test.ts b/pnpr/client/test/sharedSideEffects.test.ts index 5bbe1149ab..33368a1056 100644 --- a/pnpr/client/test/sharedSideEffects.test.ts +++ b/pnpr/client/test/sharedSideEffects.test.ts @@ -314,7 +314,7 @@ describe('signed shared artifacts', () => { }, }) expect(mismatchedPackage).toEqual(new Map()) - const selected = await resolveSharedSideEffects({ + const resolveOptions = { registryUrl, authorization: 'Bearer token', candidates: [{ @@ -332,13 +332,24 @@ describe('signed shared artifacts', () => { trustedKeys: { 'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'), }, - }) + } + const selected = await resolveSharedSideEffects(resolveOptions) expect(selected.get(payload().inputKey)?.payload).toEqual(payload()) expect(selected.get(payload().inputKey)?.envelopeDigest).toBe( [envelope, alternateEnvelope] .map(signedArtifactEnvelopeDigest) .sort()[0] ) + const pinnedDigest = signedArtifactEnvelopeDigest(alternateEnvelope) + const pinned = await resolveSharedSideEffects({ + ...resolveOptions, + pinnedEnvelopeDigests: new Map([[payload().inputKey, pinnedDigest]]), + }) + expect(pinned.get(payload().inputKey)?.envelopeDigest).toBe(pinnedDigest) + await expect(resolveSharedSideEffects({ + ...resolveOptions, + pinnedEnvelopeDigests: new Map([[payload().inputKey, '0'.repeat(64)]]), + })).resolves.toEqual(new Map()) await expect(downloadSharedArtifactBlob({ registryUrl, authorization: 'Bearer token', @@ -348,6 +359,8 @@ describe('signed shared artifacts', () => { '/-/pnpr/v0/artifacts', '/-/pnpr/v0/artifacts/resolve', '/-/pnpr/v0/artifacts/resolve', + '/-/pnpr/v0/artifacts/resolve', + '/-/pnpr/v0/artifacts/resolve', '/-/pnpr/v0/artifacts/blob', ]) expect(requests.every(request => request.authorization === 'Bearer token')).toBe(true)