feat: pin shared build artifacts in the lockfile (#14243)
Persist verified shared build artifact envelope digests in the package snapshot that consumed them. Scope each pin by dependency input key, artifact owner, and consumer platform so compatible variants remain independently reproducible. Apply pins only after the existing trust and compatibility checks. A missing pinned artifact is a remote-cache miss and falls back to a local build; frozen installs enforce pins without mutating the wanted lockfile. Add `pnpm update --build-artifacts` as an explicit update mode that clears existing pins, forces materialization, and records only artifacts that complete verification and hydration without changing package versions. Reject command combinations that cannot safely write or verify the lockfile. Implement the lockfile format, install plumbing, update mode, and tests in both TypeScript pnpm and pacquet. Related to pnpm/pnpm#13771.
This commit is contained in:
1 parent
aaaca889de
commit
5dd145394b
39 files changed
+992
-100
No files matched your search
@@ -0,0 +1,13 @@
|
||||
---
|
||||
"@pnpm/lockfile.types": minor
|
||||
"@pnpm/lockfile.fs": patch
|
||||
"@pnpm/pnpr.client": minor
|
||||
"@pnpm/installing.deps-resolver": patch
|
||||
"@pnpm/installing.deps-restorer": minor
|
||||
"@pnpm/installing.deps-installer": minor
|
||||
"@pnpm/installing.commands": minor
|
||||
"pnpm": minor
|
||||
"pacquet": minor
|
||||
---
|
||||
|
||||
Remote build artifacts are now pinned in `pnpm-lock.yaml` by input key, owner, and consumer platform. Frozen installs enforce existing pins without changing them. Run `pnpm update --build-artifacts` to explicitly replace the recorded pins ([pnpm/pnpm#13771](https://github.com/pnpm/pnpm/issues/13771)).
|
||||
@@ -244,6 +244,9 @@ pub struct InstallArgs {
|
||||
#[clap(long = "update-checksums")]
|
||||
pub update_checksums: bool,
|
||||
|
||||
#[clap(skip)]
|
||||
pub(crate) refresh_artifact_pins: bool,
|
||||
|
||||
/// Maximum number of concurrent network requests during install.
|
||||
#[clap(long = "network-concurrency")]
|
||||
pub network_concurrency: Option<usize>,
|
||||
@@ -327,6 +330,7 @@ impl InstallArgs {
|
||||
trust_lockfile: false,
|
||||
no_trust_lockfile: false,
|
||||
update_checksums: false,
|
||||
refresh_artifact_pins: false,
|
||||
network_concurrency: None,
|
||||
fetch_timeout: None,
|
||||
fetch_warn_timeout_ms: None,
|
||||
@@ -336,6 +340,16 @@ impl InstallArgs {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn for_build_artifact_update(
|
||||
supported_architectures: SupportedArchitecturesArgs,
|
||||
) -> Self {
|
||||
let mut args = Self::for_reresolving_install();
|
||||
args.supported_architectures = supported_architectures;
|
||||
args.force = true;
|
||||
args.refresh_artifact_pins = true;
|
||||
args
|
||||
}
|
||||
|
||||
/// Run the repeat-install fast path before any of the async install
|
||||
/// machinery exists: when every gate below holds and
|
||||
/// [`install_already_up_to_date`] confirms nothing changed since the
|
||||
@@ -366,6 +380,7 @@ impl InstallArgs {
|
||||
if self.effective_frozen_lockfile(config)
|
||||
|| self.lockfile_only
|
||||
|| self.force
|
||||
|| self.refresh_artifact_pins
|
||||
|| self.verify_deps_before_run_install
|
||||
{
|
||||
return false;
|
||||
@@ -495,6 +510,7 @@ impl InstallArgs {
|
||||
) -> miette::Result<()> {
|
||||
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
|
||||
let frozen_lockfile = match self.configured_frozen_lockfile(state.config) {
|
||||
_ if self.refresh_artifact_pins => false,
|
||||
Some(value) => value,
|
||||
None if state.config.ci
|
||||
&& !self.lockfile_only
|
||||
@@ -550,6 +566,7 @@ impl InstallArgs {
|
||||
trust_lockfile,
|
||||
no_trust_lockfile,
|
||||
update_checksums,
|
||||
refresh_artifact_pins,
|
||||
network_concurrency: _,
|
||||
fetch_timeout: _,
|
||||
fetch_warn_timeout_ms: _,
|
||||
@@ -566,7 +583,9 @@ impl InstallArgs {
|
||||
// mutual `overrides_with` collapses both spellings to the
|
||||
// last-specified, so at most one is set and the precedence here
|
||||
// is straightforward.
|
||||
let prefer_frozen_lockfile = if prefer_frozen_lockfile {
|
||||
let prefer_frozen_lockfile = if refresh_artifact_pins {
|
||||
Some(false)
|
||||
} else if prefer_frozen_lockfile {
|
||||
Some(true)
|
||||
} else if no_prefer_frozen_lockfile {
|
||||
Some(false)
|
||||
@@ -607,7 +626,8 @@ impl InstallArgs {
|
||||
// pnpr fast path: when a `pnprServer` URL is configured, offload
|
||||
// resolution + fetching to it, then link `node_modules` from the
|
||||
// server-produced lockfile via the normal frozen install.
|
||||
if let Some(pnpr_server) = config.pnpr_server.as_deref() {
|
||||
if let Some(pnpr_server) = config.pnpr_server.as_deref().filter(|_| !refresh_artifact_pins)
|
||||
{
|
||||
// The pnpr path resolves and links through the server, so it
|
||||
// can't honor `--dry-run`'s no-write contract. Reject up front,
|
||||
// mirroring pnpm's CONFIG_CONFLICT_DRY_RUN_WITH_PNPR_SERVER.
|
||||
@@ -639,6 +659,28 @@ impl InstallArgs {
|
||||
.await;
|
||||
}
|
||||
|
||||
if refresh_artifact_pins && !config.lockfile {
|
||||
return Err(BuildArtifactsWithoutLockfile.into());
|
||||
}
|
||||
let mut refreshed_lockfile = None;
|
||||
if refresh_artifact_pins {
|
||||
refreshed_lockfile = lockfile
|
||||
.get()
|
||||
.map_err(|error| miette::Report::new(error).wrap_err("load the lockfile"))?
|
||||
.cloned();
|
||||
if let Some(snapshots) =
|
||||
refreshed_lockfile.as_mut().and_then(|lockfile| lockfile.snapshots.as_mut())
|
||||
{
|
||||
for snapshot in snapshots.values_mut() {
|
||||
snapshot.clear_artifact_pins();
|
||||
}
|
||||
}
|
||||
}
|
||||
let install_lockfile = if refresh_artifact_pins {
|
||||
MaybeLazyLockfile::Loaded(refreshed_lockfile.as_ref())
|
||||
} else {
|
||||
MaybeLazyLockfile::Lazy(lockfile)
|
||||
};
|
||||
let install = Install {
|
||||
tarball_mem_cache: std::sync::Arc::clone(tarball_mem_cache),
|
||||
http_client,
|
||||
@@ -646,7 +688,7 @@ impl InstallArgs {
|
||||
config,
|
||||
manifest,
|
||||
emit_initial_manifest: true,
|
||||
lockfile: MaybeLazyLockfile::Lazy(lockfile),
|
||||
lockfile: install_lockfile,
|
||||
lockfile_path: Some(&lockfile_path),
|
||||
dependency_groups: dependency_options.dependency_groups(config.optional),
|
||||
frozen_lockfile,
|
||||
@@ -773,6 +815,11 @@ struct FrozenStoreIncompatibleWithPnpr;
|
||||
)]
|
||||
struct DryRunIncompatibleWithPnpr;
|
||||
|
||||
#[derive(Debug, Display, Error, Diagnostic)]
|
||||
#[display("Cannot update build artifacts when lockfile is set to false")]
|
||||
#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE))]
|
||||
struct BuildArtifactsWithoutLockfile;
|
||||
|
||||
fn resolve_project(
|
||||
dir: String,
|
||||
manifest: &pnpm_package_manifest::PackageManifest,
|
||||
@@ -1119,25 +1166,13 @@ async fn install_via_pnpr_inner<Reporter: self::Reporter + 'static>(
|
||||
workspace_projects_override: None,
|
||||
};
|
||||
|
||||
let result = match (selection, lockfile_verification_override) {
|
||||
(Some(selection), Some(lockfile_verification_override)) => {
|
||||
Box::pin(install.run_selected_with_lockfile_verification::<Reporter>(
|
||||
workspace_install_selection(selection),
|
||||
lockfile_verification_override,
|
||||
))
|
||||
.await
|
||||
}
|
||||
(Some(selection), None) => {
|
||||
Box::pin(install.run_selected::<Reporter>(workspace_install_selection(selection)))
|
||||
.await
|
||||
}
|
||||
(None, Some(lockfile_verification_override)) => {
|
||||
install
|
||||
.run_with_lockfile_verification::<Reporter>(lockfile_verification_override)
|
||||
.await
|
||||
}
|
||||
(None, None) => install.run::<Reporter>().await,
|
||||
};
|
||||
let result = install
|
||||
.run_with_artifact_pin_recording::<Reporter>(
|
||||
selection.map(workspace_install_selection),
|
||||
lockfile_verification_override,
|
||||
!link.frozen_lockfile,
|
||||
)
|
||||
.await;
|
||||
// On failure the prefetcher is dropped, not shut down: shutdown
|
||||
// waits for every in-flight prefetch download (each task holds a
|
||||
// store-index writer handle), which would hold the fail-fast
|
||||
@@ -1353,13 +1388,14 @@ async fn install_via_pnpr_inner<Reporter: self::Reporter + 'static>(
|
||||
pnpmfile_hook_override: pnpmfile_hook,
|
||||
workspace_projects_override: None,
|
||||
};
|
||||
match selection {
|
||||
Some(selection) => {
|
||||
Box::pin(install.run_selected::<Reporter>(workspace_install_selection(selection))).await
|
||||
}
|
||||
None => install.run::<Reporter>().await,
|
||||
}
|
||||
.wrap_err("linking dependencies resolved via the pnpr server")?;
|
||||
install
|
||||
.run_with_artifact_pin_recording::<Reporter>(
|
||||
selection.map(workspace_install_selection),
|
||||
None,
|
||||
!link.frozen_lockfile,
|
||||
)
|
||||
.await
|
||||
.wrap_err("linking dependencies resolved via the pnpr server")?;
|
||||
|
||||
// The materialization install has awaited every tarball's mem-cache
|
||||
// slot, so all prefetch downloads have finished and queued their
|
||||
|
||||
@@ -712,7 +712,7 @@ pub(crate) fn apply_install_cli_config(cfg: &mut Config, args: &InstallArgs) {
|
||||
cfg.ignore_scripts =
|
||||
resolve_bool_override(args.ignore_scripts, args.no_ignore_scripts, cfg.ignore_scripts);
|
||||
cfg.ignore_pnpmfile = args.ignore_pnpmfile || cfg.ignore_pnpmfile;
|
||||
cfg.force = args.force || cfg.force;
|
||||
cfg.force = args.force || args.refresh_artifact_pins || cfg.force;
|
||||
if let Some(network_concurrency) = args.network_concurrency {
|
||||
cfg.network_concurrency = network_concurrency;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::{
|
||||
State,
|
||||
cli_args::{
|
||||
install::InstallArgs,
|
||||
lockfile_dir::LockfileDirArg,
|
||||
pipelines::InstallFamilySelection,
|
||||
recursive,
|
||||
@@ -94,6 +95,10 @@ pub struct UpdateArgs {
|
||||
#[clap(long)]
|
||||
pub patches: bool,
|
||||
|
||||
/// Update remote build artifacts without changing package versions.
|
||||
#[clap(long = "build-artifacts")]
|
||||
pub build_artifacts: bool,
|
||||
|
||||
/// Write the resolved version without a range operator when
|
||||
/// rewriting the manifest under `--latest`.
|
||||
#[clap(short = 'E', long = "save-exact")]
|
||||
@@ -177,9 +182,22 @@ enum WorkspaceUpdateError {
|
||||
#[diagnostic(code(ERR_PNPM_PATCHES_WITH_SELECTOR))]
|
||||
struct PatchesWithSelectorError;
|
||||
|
||||
#[derive(Debug, Display, Error, Diagnostic)]
|
||||
#[display(
|
||||
"--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches"
|
||||
)]
|
||||
#[diagnostic(code(ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR))]
|
||||
struct BuildArtifactsWithSelectorError;
|
||||
|
||||
#[derive(Debug, Display, Error, Diagnostic)]
|
||||
#[display("Cannot update build artifacts without downloading and verifying them")]
|
||||
#[diagnostic(code(ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY))]
|
||||
struct BuildArtifactsWithLockfileOnlyError;
|
||||
|
||||
impl UpdateArgs {
|
||||
pub(crate) fn apply_cli_config(&self, config: &mut Config) {
|
||||
config.ignore_pnpmfile = self.ignore_pnpmfile || config.ignore_pnpmfile;
|
||||
config.force = self.build_artifacts || config.force;
|
||||
if let Some(pnpr_server) = self.pnpr_server.clone() {
|
||||
config.pnpr_server = Some(pnpr_server);
|
||||
}
|
||||
@@ -190,7 +208,13 @@ impl UpdateArgs {
|
||||
mut state: State,
|
||||
) -> miette::Result<()> {
|
||||
self.check_patches_options()?;
|
||||
self.check_build_artifacts_options()?;
|
||||
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
|
||||
if self.build_artifacts {
|
||||
return InstallArgs::for_build_artifact_update(self.supported_architectures.clone())
|
||||
.run::<Reporter>(state)
|
||||
.await;
|
||||
}
|
||||
let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?;
|
||||
let include_direct = self.dependency_options.include_direct();
|
||||
let update_actions = self.should_update_github_actions(state.config, &include_direct);
|
||||
@@ -320,7 +344,13 @@ impl UpdateArgs {
|
||||
selection: InstallFamilySelection,
|
||||
) -> miette::Result<()> {
|
||||
self.check_patches_options()?;
|
||||
self.check_build_artifacts_options()?;
|
||||
state.http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
|
||||
if self.build_artifacts {
|
||||
return InstallArgs::for_build_artifact_update(self.supported_architectures.clone())
|
||||
.run_selected::<Reporter>(state, selection)
|
||||
.await;
|
||||
}
|
||||
let workspace_root = self.check_workspace_option(state.config.workspace_dir.as_deref())?;
|
||||
let include_direct = self.dependency_options.include_direct();
|
||||
let update_actions = self.should_update_github_actions(state.config, &include_direct);
|
||||
@@ -455,6 +485,7 @@ impl UpdateArgs {
|
||||
config: &'static Config,
|
||||
) -> miette::Result<()> {
|
||||
self.check_patches_options()?;
|
||||
self.check_build_artifacts_options()?;
|
||||
self.check_workspace_option(None)?;
|
||||
let selected_hashes: Option<HashSet<String>> = if self.interactive {
|
||||
match crate::cli_args::update_interactive::select_global_package_groups(
|
||||
@@ -515,6 +546,24 @@ impl UpdateArgs {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_build_artifacts_options(&self) -> miette::Result<()> {
|
||||
if !self.build_artifacts {
|
||||
return Ok(());
|
||||
}
|
||||
if !self.packages.is_empty()
|
||||
|| self.latest
|
||||
|| self.interactive
|
||||
|| self.global
|
||||
|| self.patches
|
||||
{
|
||||
return Err(BuildArtifactsWithSelectorError.into());
|
||||
}
|
||||
if self.lockfile_only {
|
||||
return Err(BuildArtifactsWithLockfileOnlyError.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn can_delegate_patch_refresh(
|
||||
&self,
|
||||
update_actions: bool,
|
||||
|
||||
@@ -160,6 +160,44 @@ fn patches_is_a_selectorless_update_mode() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_artifacts_is_a_selectorless_update_mode() {
|
||||
let build_artifacts = update_args(&["--build-artifacts"]);
|
||||
assert!(build_artifacts.build_artifacts);
|
||||
build_artifacts.check_build_artifacts_options().expect("standalone --build-artifacts");
|
||||
|
||||
for args in [
|
||||
&["--build-artifacts", "foo"][..],
|
||||
&["--build-artifacts", "--latest"][..],
|
||||
&["--build-artifacts", "--interactive"][..],
|
||||
&["--build-artifacts", "--global"][..],
|
||||
&["--build-artifacts", "--patches"][..],
|
||||
] {
|
||||
let error = update_args(args)
|
||||
.check_build_artifacts_options()
|
||||
.expect_err("--build-artifacts combination must fail");
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches",
|
||||
);
|
||||
}
|
||||
|
||||
let error = update_args(&["--build-artifacts", "--lockfile-only"])
|
||||
.check_build_artifacts_options()
|
||||
.expect_err("--build-artifacts --lockfile-only must fail");
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"Cannot update build artifacts without downloading and verifying them",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_artifacts_forces_reinstallation() {
|
||||
let mut config = Config::default();
|
||||
update_args(&["--build-artifacts"]).apply_cli_config(&mut config);
|
||||
assert!(config.force);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constrained_patch_refresh_stays_on_the_client() {
|
||||
let all_groups = vec![DependencyGroup::Prod, DependencyGroup::Dev, DependencyGroup::Optional];
|
||||
|
||||
@@ -25,6 +25,53 @@ const HAS_PRERELEASE: &str = "@pnpm.e2e/has-prerelease";
|
||||
/// indirect-dependency update behavior when the direct dep is ignored.
|
||||
const PARENT: &str = "@pnpm.e2e/pkg-with-1-dep";
|
||||
|
||||
#[test]
|
||||
fn build_artifacts_clears_existing_pins_and_overrides_configured_frozen_lockfile() {
|
||||
let (root, workspace, npmrc_info) = setup();
|
||||
let AddMockedRegistry { mock_instance, .. } = npmrc_info;
|
||||
write_manifest(&workspace, r#"{ "is-positive": "1.0.0" }"#);
|
||||
pacquet(&workspace, ["install"]).assert().success();
|
||||
|
||||
let wanted_path = workspace.join("pnpm-lock.yaml");
|
||||
let mut wanted = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace)
|
||||
.expect("load wanted lockfile")
|
||||
.expect("wanted lockfile");
|
||||
wanted
|
||||
.snapshots
|
||||
.as_mut()
|
||||
.and_then(|snapshots| snapshots.values_mut().next())
|
||||
.expect("dependency snapshot")
|
||||
.record_artifact_pin(
|
||||
"dependency-side-effects:v1:deps=old".to_string(),
|
||||
"organization:acme".to_string(),
|
||||
"platform".to_string(),
|
||||
"0".repeat(64),
|
||||
);
|
||||
wanted.save_to_path(&wanted_path).expect("save artifact pin");
|
||||
|
||||
let workspace_yaml_path = workspace.join("pnpm-workspace.yaml");
|
||||
let mut workspace_yaml =
|
||||
fs::read_to_string(&workspace_yaml_path).expect("read workspace settings");
|
||||
workspace_yaml.push_str("frozenLockfile: true\n");
|
||||
fs::write(workspace_yaml_path, workspace_yaml).expect("write workspace settings");
|
||||
|
||||
pacquet(&workspace, ["update", "--build-artifacts"]).assert().success();
|
||||
|
||||
let refreshed = pnpm_lockfile::Lockfile::load_wanted_from_dir(&workspace)
|
||||
.expect("load refreshed lockfile")
|
||||
.expect("refreshed lockfile");
|
||||
assert!(
|
||||
refreshed
|
||||
.snapshots
|
||||
.as_ref()
|
||||
.into_iter()
|
||||
.flat_map(|snapshots| snapshots.values())
|
||||
.all(|snapshot| snapshot.artifact_pins.is_none()),
|
||||
);
|
||||
|
||||
drop((root, mock_instance));
|
||||
}
|
||||
|
||||
/// Spin up a temp workspace with the mocked registry and return the
|
||||
/// pieces a multi-step update test needs.
|
||||
fn setup() -> (TempDir, std::path::PathBuf, AddMockedRegistry) {
|
||||
|
||||
@@ -35,6 +35,7 @@ fn snap(deps: &[(&str, &str)]) -> SnapshotEntry {
|
||||
id: None,
|
||||
dependencies: (!map.is_empty()).then_some(map),
|
||||
optional_dependencies: None,
|
||||
artifact_pins: None,
|
||||
transitive_peer_dependencies: None,
|
||||
patched: None,
|
||||
optional: false,
|
||||
|
||||
@@ -120,6 +120,16 @@ pub struct CreateVirtualStoreOutput {
|
||||
/// the bridge into the link phase instead). Pacquet decouples
|
||||
/// fetch and walk, so the index is built here at fetch time.
|
||||
pub cas_paths_by_pkg_id: Option<CasPathsByPkgId>,
|
||||
pub artifact_pin_records: Vec<ArtifactPinRecord>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ArtifactPinRecord {
|
||||
pub snapshot_key: PackageKey,
|
||||
pub input_key: String,
|
||||
pub owner: String,
|
||||
pub platform_fingerprint: String,
|
||||
pub envelope_digest: String,
|
||||
}
|
||||
|
||||
/// This subroutine generates filesystem layout for the virtual store at `node_modules/.pacquet`.
|
||||
@@ -304,6 +314,7 @@ impl CreateVirtualStore<'_> {
|
||||
materialized_snapshots: Vec::new(),
|
||||
fetch_failed: HashSet::new(),
|
||||
cas_paths_by_pkg_id: is_hoisted.then(CasPathsByPkgId::new),
|
||||
artifact_pin_records: Vec::new(),
|
||||
});
|
||||
};
|
||||
let packages = packages.ok_or(CreateVirtualStoreError::MissingPackagesSection)?;
|
||||
@@ -856,7 +867,7 @@ impl CreateVirtualStore<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
crate::shared_side_effects::apply_shared_side_effects(
|
||||
let artifact_pin_records = crate::shared_side_effects::apply_shared_side_effects(
|
||||
config,
|
||||
snapshots,
|
||||
packages,
|
||||
@@ -919,6 +930,7 @@ impl CreateVirtualStore<'_> {
|
||||
materialized_snapshots,
|
||||
fetch_failed,
|
||||
cas_paths_by_pkg_id,
|
||||
artifact_pin_records,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use crate::{
|
||||
AllowBuildPolicy, BuildModules, BuildModulesError, CreateVirtualStore, CreateVirtualStoreError,
|
||||
CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies, LinkHoistedModulesError,
|
||||
LinkHoistedModulesOpts, LinkRootComponentMembersError, LinkVirtualStoreBinsError,
|
||||
LockfileToHoistedDepGraphOptions, SkippedSnapshots, SymlinkDirectDependencies,
|
||||
SymlinkDirectDependenciesError, SymlinkPackageError, VersionPolicyError, VirtualStoreLayout,
|
||||
any_installability_constraint, build_direct_deps_by_importer, direct_dep_names_for_importer,
|
||||
get_hoisted_dependencies, link_hoisted_modules, link_top_level_bins,
|
||||
lockfile_to_hoisted_dep_graph, symlink_direct_dependencies::importer_root_dir,
|
||||
AllowBuildPolicy, ArtifactPinRecord, BuildModules, BuildModulesError, CreateVirtualStore,
|
||||
CreateVirtualStoreError, CreateVirtualStoreOutput, HoistedDepGraphError, HoistedDependencies,
|
||||
LinkHoistedModulesError, LinkHoistedModulesOpts, LinkRootComponentMembersError,
|
||||
LinkVirtualStoreBinsError, LockfileToHoistedDepGraphOptions, SkippedSnapshots,
|
||||
SymlinkDirectDependencies, SymlinkDirectDependenciesError, SymlinkPackageError,
|
||||
VersionPolicyError, VirtualStoreLayout, any_installability_constraint,
|
||||
build_direct_deps_by_importer, direct_dep_names_for_importer, get_hoisted_dependencies,
|
||||
link_hoisted_modules, link_top_level_bins, lockfile_to_hoisted_dep_graph,
|
||||
symlink_direct_dependencies::importer_root_dir,
|
||||
};
|
||||
|
||||
mod build_phase;
|
||||
@@ -668,6 +669,7 @@ where
|
||||
materialized_snapshots,
|
||||
fetch_failed,
|
||||
cas_paths_by_pkg_id,
|
||||
artifact_pin_records,
|
||||
} = {
|
||||
let mut verify_fut = std::pin::pin!(verify_fut);
|
||||
let mut create_virtual_store_fut = std::pin::pin!(create_virtual_store_fut);
|
||||
@@ -879,6 +881,7 @@ where
|
||||
ignored_builds,
|
||||
deferred_builds,
|
||||
store_index_teardown: writer_task,
|
||||
artifact_pin_records,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -889,6 +892,7 @@ where
|
||||
/// land without churning every call site.
|
||||
#[derive(Debug)]
|
||||
pub struct InstallFrozenLockfileOutput {
|
||||
pub artifact_pin_records: Vec<ArtifactPinRecord>,
|
||||
/// Hoisted-dependencies map produced by the isolated-linker
|
||||
/// hoist pass — empty when both hoist patterns are `None` and
|
||||
/// always empty under `nodeLinker: hoisted` (the hoisted
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::{
|
||||
AllowBuildPolicy, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot, build_deps_subgraph,
|
||||
deps_graph::in_lockfile_order, install_frozen_lockfile::find_runtime_node_major,
|
||||
AllowBuildPolicy, ArtifactPinRecord, RequiresBuildBySnapshot, SideEffectsMapsBySnapshot,
|
||||
build_deps_subgraph, deps_graph::in_lockfile_order,
|
||||
install_frozen_lockfile::find_runtime_node_major,
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use pnpm_config::Config;
|
||||
@@ -10,7 +11,7 @@ use pnpm_pnpr_client::{
|
||||
ArtifactManifest, ArtifactPayload, BuilderProfile, CompatibilityConstraints,
|
||||
LinuxGlibcPlatform, OwnerScope, PackageIdentity, PnprClient, PublishArtifactRequest,
|
||||
ResolveArtifactsOptions, SignedArtifactEnvelope, blob_id, linux_glibc_supported_tags,
|
||||
linux_glibc_tag,
|
||||
linux_glibc_tag, platform_fingerprint,
|
||||
};
|
||||
use sha2::{Digest as _, Sha512};
|
||||
use std::{
|
||||
@@ -36,6 +37,7 @@ pub struct SharedSideEffectsPublisher {
|
||||
|
||||
struct CandidateGroup {
|
||||
candidate: ArtifactCandidate,
|
||||
pinned_envelope_digest: Option<String>,
|
||||
snapshots: Vec<(PackageKey, String)>,
|
||||
}
|
||||
|
||||
@@ -47,25 +49,34 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
allow_build_policy: &AllowBuildPolicy,
|
||||
base_cas_paths: &BaseCasPaths,
|
||||
side_effects_maps_by_snapshot: &mut SideEffectsMapsBySnapshot,
|
||||
) {
|
||||
) -> Vec<ArtifactPinRecord> {
|
||||
if config.ignore_scripts || config.frozen_store {
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
let (Some(server), Some(settings)) =
|
||||
(config.pnpr_server.as_deref(), config.remote_side_effects_cache.as_ref())
|
||||
else {
|
||||
return;
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(platform) = linux_glibc_platform(snapshots) else { return };
|
||||
let Some(platform) = linux_glibc_platform(snapshots) else { return Vec::new() };
|
||||
let supported_tags = match linux_glibc_supported_tags(platform) {
|
||||
Ok(tags) => tags,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform is unsupported");
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let Some(trusted_keys) = decoded_trusted_keys(settings) else { return Vec::new() };
|
||||
let Some(organization) = non_empty(&settings.organization) else { return Vec::new() };
|
||||
let owner = OwnerScope::organization(organization.to_string());
|
||||
let owner_namespace = owner.namespace();
|
||||
let fingerprint = match platform_fingerprint(&supported_tags) {
|
||||
Ok(fingerprint) => fingerprint,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "pacquet::install", %error, "remote side-effects platform fingerprint is invalid");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let Some(trusted_keys) = decoded_trusted_keys(settings) else { return };
|
||||
let Some(organization) = non_empty(&settings.organization) else { return };
|
||||
|
||||
let eligible_packages: HashSet<String> = settings.packages.iter().cloned().collect();
|
||||
let roots: Vec<PackageKey> = in_lockfile_order(snapshots)
|
||||
@@ -84,7 +95,7 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
"planned remote side-effects candidates",
|
||||
);
|
||||
if roots.is_empty() {
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
let graph = build_deps_subgraph(snapshots, packages, roots.clone());
|
||||
let mut deps_state_cache = pnpm_graph_hasher::DepsStateCache::new();
|
||||
@@ -137,8 +148,15 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
version: package_version(&metadata_key, metadata.version.as_deref()),
|
||||
},
|
||||
source_integrity,
|
||||
owner: OwnerScope::organization(organization.to_string()),
|
||||
owner: owner.clone(),
|
||||
};
|
||||
let pinned_envelope_digest = snapshots
|
||||
.get(&snapshot_key)
|
||||
.and_then(|snapshot| snapshot.artifact_pins.as_ref())
|
||||
.and_then(|pins| pins.get(&input_key))
|
||||
.and_then(|owners| owners.get(&owner_namespace))
|
||||
.and_then(|platforms| platforms.get(&fingerprint))
|
||||
.cloned();
|
||||
if let Some(group) = groups.get_mut(&input_key) {
|
||||
if group.candidate.package != candidate.package
|
||||
|| group.candidate.source_integrity != candidate.source_integrity
|
||||
@@ -147,16 +165,30 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
collisions.insert(input_key);
|
||||
continue;
|
||||
}
|
||||
if group.pinned_envelope_digest.is_some()
|
||||
&& pinned_envelope_digest.is_some()
|
||||
&& group.pinned_envelope_digest != pinned_envelope_digest
|
||||
{
|
||||
groups.remove(&input_key);
|
||||
collisions.insert(input_key);
|
||||
continue;
|
||||
}
|
||||
group.pinned_envelope_digest =
|
||||
group.pinned_envelope_digest.take().or(pinned_envelope_digest);
|
||||
group.snapshots.push((snapshot_key, local_cache_key));
|
||||
} else {
|
||||
groups.insert(
|
||||
input_key,
|
||||
CandidateGroup { candidate, snapshots: vec![(snapshot_key, local_cache_key)] },
|
||||
CandidateGroup {
|
||||
candidate,
|
||||
pinned_envelope_digest,
|
||||
snapshots: vec![(snapshot_key, local_cache_key)],
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if groups.is_empty() {
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "pacquet::install",
|
||||
@@ -167,11 +199,17 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
let client = PnprClient::new(server);
|
||||
if let Err(error) = client.handshake_artifacts().await {
|
||||
tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache handshake failed");
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
let authorization = config.auth_headers.for_url(server);
|
||||
let allowed_builds =
|
||||
groups.values().map(|group| group.candidate.package.name.clone()).collect();
|
||||
let pinned_envelope_digests = groups
|
||||
.iter()
|
||||
.filter_map(|(input_key, group)| {
|
||||
group.pinned_envelope_digest.as_ref().map(|digest| (input_key.clone(), digest.clone()))
|
||||
})
|
||||
.collect();
|
||||
let resolved = match client
|
||||
.resolve_artifacts(ResolveArtifactsOptions {
|
||||
candidates: groups.values().map(|group| group.candidate.clone()).collect(),
|
||||
@@ -180,6 +218,7 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
allowed_builds,
|
||||
ignore_scripts: false,
|
||||
trusted_keys,
|
||||
pinned_envelope_digests,
|
||||
authorization: authorization.clone(),
|
||||
})
|
||||
.await
|
||||
@@ -187,10 +226,20 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "pacquet::install", %error, "remote side-effects cache lookup failed");
|
||||
return;
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
for (input_key, group) in &groups {
|
||||
if group.pinned_envelope_digest.is_some() && !resolved.contains_key(input_key) {
|
||||
tracing::warn!(
|
||||
target: "pacquet::install",
|
||||
package = %group.candidate.package.name,
|
||||
"pinned remote side-effects artifact is unavailable; building locally",
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut artifact_pin_records = Vec::new();
|
||||
for (input_key, artifact) in resolved {
|
||||
let Some(group) = groups.get(&input_key) else { continue };
|
||||
let Some((first_snapshot, _)) = group.snapshots.first() else { continue };
|
||||
@@ -274,8 +323,16 @@ pub(crate) async fn apply_shared_side_effects(
|
||||
.map_or_else(HashMap::new, |maps| (**maps).clone());
|
||||
maps.insert(local_cache_key.clone(), overlay.clone());
|
||||
side_effects_maps_by_snapshot.insert(snapshot_key.clone(), Arc::new(maps));
|
||||
artifact_pin_records.push(ArtifactPinRecord {
|
||||
snapshot_key: snapshot_key.clone(),
|
||||
input_key: input_key.clone(),
|
||||
owner: owner_namespace.clone(),
|
||||
platform_fingerprint: fingerprint.clone(),
|
||||
envelope_digest: artifact.envelope_digest.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
artifact_pin_records
|
||||
}
|
||||
|
||||
/// Decode the configured trust root, or `None` when it is absent or unusable.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::{PkgName, SnapshotDepRef};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -27,6 +27,9 @@ pub struct SnapshotEntry {
|
||||
)]
|
||||
pub optional_dependencies: Option<HashMap<PkgName, SnapshotDepRef>>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub artifact_pins: Option<ArtifactPins>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub transitive_peer_dependencies: Option<Vec<String>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -45,6 +48,36 @@ pub struct SnapshotEntry {
|
||||
pub optional: bool,
|
||||
}
|
||||
|
||||
pub type ArtifactPins = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
|
||||
|
||||
impl SnapshotEntry {
|
||||
pub fn record_artifact_pin(
|
||||
&mut self,
|
||||
input_key: String,
|
||||
owner: String,
|
||||
platform_fingerprint: String,
|
||||
envelope_digest: String,
|
||||
) -> bool {
|
||||
let previous = self
|
||||
.artifact_pins
|
||||
.as_ref()
|
||||
.and_then(|pins| pins.get(&input_key))
|
||||
.and_then(|owners| owners.get(&owner))
|
||||
.and_then(|platforms| platforms.get(&platform_fingerprint));
|
||||
if previous == Some(&envelope_digest) {
|
||||
return false;
|
||||
}
|
||||
let pins = self.artifact_pins.get_or_insert_default();
|
||||
let owners = pins.entry(input_key).or_default();
|
||||
owners.entry(owner).or_default().insert(platform_fingerprint, envelope_digest);
|
||||
true
|
||||
}
|
||||
|
||||
pub fn clear_artifact_pins(&mut self) -> bool {
|
||||
self.artifact_pins.take().is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(
|
||||
clippy::trivially_copy_pass_by_ref,
|
||||
reason = "serde skip_serializing_if is called as f(&field)"
|
||||
|
||||
@@ -34,3 +34,32 @@ fn optional_defaults_false_and_omits_when_false() {
|
||||
"the `optional` key must not be serialized when false:\n{out}",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_pins_round_trip_and_preserve_other_inputs() {
|
||||
let yaml = text_block! {
|
||||
"artifactPins:"
|
||||
" dependency-side-effects:v1:deps=old:"
|
||||
" organization:acme:"
|
||||
" linux-node22: abc123"
|
||||
};
|
||||
let mut entry: SnapshotEntry = serde_saphyr::from_str(yaml).expect("parse");
|
||||
assert!(!entry.record_artifact_pin(
|
||||
"dependency-side-effects:v1:deps=old".to_string(),
|
||||
"organization:acme".to_string(),
|
||||
"linux-node22".to_string(),
|
||||
"abc123".to_string(),
|
||||
));
|
||||
assert!(entry.record_artifact_pin(
|
||||
"dependency-side-effects:v1:deps=new".to_string(),
|
||||
"organization:acme".to_string(),
|
||||
"linux-node22".to_string(),
|
||||
"def456".to_string(),
|
||||
));
|
||||
let out = serialize_yaml::to_string(&entry).expect("serialize");
|
||||
assert!(out.contains("dependency-side-effects:v1:deps=old:"));
|
||||
assert!(out.contains("dependency-side-effects:v1:deps=new:"));
|
||||
assert!(out.contains("linux-node22: def456"));
|
||||
assert!(entry.clear_artifact_pins());
|
||||
assert!(!entry.clear_artifact_pins());
|
||||
}
|
||||
@@ -37,7 +37,7 @@ const INDENT: usize = 2;
|
||||
|
||||
/// Per-package / per-snapshot key priority, matching pnpm's lockfile
|
||||
/// key sort.
|
||||
const ORDERED_KEYS: [&str; 20] = [
|
||||
const ORDERED_KEYS: [&str; 21] = [
|
||||
"resolution",
|
||||
"id",
|
||||
"name",
|
||||
@@ -55,6 +55,7 @@ const ORDERED_KEYS: [&str; 20] = [
|
||||
"peerDependenciesMeta",
|
||||
"dependencies",
|
||||
"optionalDependencies",
|
||||
"artifactPins",
|
||||
"transitivePeerDependencies",
|
||||
"dev",
|
||||
"optional",
|
||||
|
||||
@@ -122,6 +122,7 @@ pub fn build_package_snapshot(
|
||||
id: None,
|
||||
dependencies: (!dependencies.is_empty()).then_some(dependencies),
|
||||
optional_dependencies: None,
|
||||
artifact_pins: None,
|
||||
transitive_peer_dependencies: None,
|
||||
patched: None,
|
||||
optional: false,
|
||||
|
||||
@@ -975,6 +975,7 @@ fn build_snapshot_entry(
|
||||
id: None,
|
||||
dependencies: (!dependencies.is_empty()).then_some(dependencies),
|
||||
optional_dependencies: (!optional_dependencies.is_empty()).then_some(optional_dependencies),
|
||||
artifact_pins: None,
|
||||
transitive_peer_dependencies: (!transitive.is_empty()).then_some(transitive),
|
||||
patched: None,
|
||||
optional,
|
||||
|
||||
@@ -881,6 +881,7 @@ struct InstallRunOptions<'install, 'selection> {
|
||||
/// whose resolution belongs to a project other than the one that
|
||||
/// owns that lockfile, so the run must leave it untouched.
|
||||
save_lockfile: bool,
|
||||
record_artifact_pins: bool,
|
||||
/// pnpm's `lockfileCheck`: the caller restores the lockfile and diffs
|
||||
/// it once the install returns, so the run must leave nothing else on
|
||||
/// disk changed either. Only `pacquet dedupe --check` sets it.
|
||||
@@ -903,6 +904,7 @@ impl Default for InstallRunOptions<'_, '_> {
|
||||
lockfile_specifier_project_manifests: None,
|
||||
read_package_hooked_manifest_paths: HashSet::new(),
|
||||
save_lockfile: true,
|
||||
record_artifact_pins: false,
|
||||
lockfile_check: false,
|
||||
manifest_spec_bumps: None,
|
||||
prompt_eligibility_override: None,
|
||||
@@ -1041,6 +1043,21 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn run_with_artifact_pin_recording<Reporter: self::Reporter + 'static>(
|
||||
self,
|
||||
selection: Option<WorkspaceInstallSelection<'_>>,
|
||||
lockfile_verification_override: Option<LockfileVerificationOverride<'a>>,
|
||||
record_artifact_pins: bool,
|
||||
) -> Result<(), InstallError> {
|
||||
Box::pin(self.run_inner::<Reporter>(InstallRunOptions {
|
||||
lockfile_verification_override,
|
||||
selection,
|
||||
record_artifact_pins,
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
/// Execute the install a legacy `pacquet deploy` runs in its target
|
||||
/// directory: the deployed manifest is the root importer, while
|
||||
/// workspace discovery stays anchored at the source workspace so
|
||||
|
||||
@@ -273,6 +273,7 @@ struct CommitModulesStateInputs<'a> {
|
||||
lockfile_was_fast_updated: bool,
|
||||
save_lockfile: bool,
|
||||
loaded_wanted_lockfile: Option<&'a Lockfile>,
|
||||
fresh_wanted_lockfile: Option<&'a Lockfile>,
|
||||
}
|
||||
|
||||
fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), InstallError> {
|
||||
@@ -300,6 +301,7 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst
|
||||
lockfile_was_fast_updated,
|
||||
save_lockfile,
|
||||
loaded_wanted_lockfile,
|
||||
fresh_wanted_lockfile,
|
||||
} = inputs;
|
||||
let now = SystemTime::now();
|
||||
let effective_virtual_store_dir = config.effective_virtual_store_dir();
|
||||
@@ -464,19 +466,20 @@ fn commit_modules_state(inputs: CommitModulesStateInputs<'_>) -> Result<(), Inst
|
||||
.map_err(InstallError::SaveCurrentLockfile)?;
|
||||
}
|
||||
|
||||
// Regenerate `pnpm-lock.yaml` from the synthesized snapshot when
|
||||
// the wanted lockfile was reconstructed from
|
||||
// `<virtual_store_dir>/lock.yaml`. The no-op short-circuit above
|
||||
// handles the common case; this branch covers the rare path where
|
||||
// `.modules.yaml` was wiped or inconsistent and the frozen install
|
||||
// had to relink.
|
||||
if take_frozen_path
|
||||
&& (lockfile_synthesized_from_current
|
||||
// Persist wanted-lockfile changes discovered during frozen
|
||||
// materialization. These can be verified artifact pins, or a wanted
|
||||
// lockfile reconstructed or fast-updated before materialization.
|
||||
let frozen_wanted_lockfile_to_save = fresh_wanted_lockfile.or_else(|| {
|
||||
(lockfile_synthesized_from_current
|
||||
|| lockfile_was_fast_updated
|
||||
|| config.merge_git_branch_lockfiles)
|
||||
.then_some(loaded_wanted_lockfile)
|
||||
.flatten()
|
||||
});
|
||||
if take_frozen_path
|
||||
&& config.lockfile
|
||||
&& save_lockfile
|
||||
&& let Some(updated) = loaded_wanted_lockfile
|
||||
&& let Some(updated) = frozen_wanted_lockfile_to_save
|
||||
{
|
||||
updated
|
||||
.save_to_path(&workspace_root.join(config.wanted_lockfile_name()))
|
||||
@@ -867,6 +870,7 @@ pub(super) async fn apply_materialization_result<Reporter: self::Reporter + 'sta
|
||||
lockfile_was_fast_updated,
|
||||
save_lockfile,
|
||||
loaded_wanted_lockfile: lockfile,
|
||||
fresh_wanted_lockfile: fresh_lockfile.as_ref(),
|
||||
})?;
|
||||
|
||||
run_materialized_project_scripts::<Reporter>(MaterializedProjectScriptsInputs {
|
||||
|
||||
@@ -60,6 +60,7 @@ pub(super) struct MaterializationInputs<'a, 'install> {
|
||||
pub(super) pnpmfile_hook: Option<Arc<dyn pnpm_hooks::PnpmfileHooks>>,
|
||||
pub(super) deploy_manifest_hook: bool,
|
||||
pub(super) save_lockfile: bool,
|
||||
pub(super) record_artifact_pins: bool,
|
||||
pub(super) manifest_spec_bumps: Option<&'a crate::ManifestSpecBumps>,
|
||||
pub(super) catalogs: &'a Catalogs,
|
||||
pub(super) prefix: &'a str,
|
||||
@@ -135,6 +136,7 @@ pub(super) async fn materialize<Reporter: self::Reporter + 'static>(
|
||||
pnpmfile_hook,
|
||||
deploy_manifest_hook,
|
||||
save_lockfile,
|
||||
record_artifact_pins,
|
||||
manifest_spec_bumps,
|
||||
catalogs,
|
||||
prefix,
|
||||
@@ -300,11 +302,30 @@ pub(super) async fn materialize<Reporter: self::Reporter + 'static>(
|
||||
ignored_builds = frozen_result.ignored_builds;
|
||||
deferred_builds = frozen_result.deferred_builds;
|
||||
injected_deps = frozen_result.injected_deps;
|
||||
let fresh_lockfile =
|
||||
(record_artifact_pins && !frozen_result.artifact_pin_records.is_empty()).then(|| {
|
||||
let mut updated = lockfile.clone();
|
||||
for record in frozen_result.artifact_pin_records {
|
||||
if let Some(snapshot) = updated
|
||||
.snapshots
|
||||
.as_mut()
|
||||
.and_then(|snapshots| snapshots.get_mut(&record.snapshot_key))
|
||||
{
|
||||
snapshot.record_artifact_pin(
|
||||
record.input_key,
|
||||
record.owner,
|
||||
record.platform_fingerprint,
|
||||
record.envelope_digest,
|
||||
);
|
||||
}
|
||||
}
|
||||
updated
|
||||
});
|
||||
(
|
||||
frozen_result.hoisted_dependencies,
|
||||
frozen_result.hoisted_locations,
|
||||
frozen_result.skipped,
|
||||
None,
|
||||
fresh_lockfile,
|
||||
frozen_result.store_index_teardown,
|
||||
)
|
||||
} else {
|
||||
|
||||
@@ -72,6 +72,7 @@ where
|
||||
lockfile_specifier_project_manifests,
|
||||
read_package_hooked_manifest_paths,
|
||||
save_lockfile,
|
||||
record_artifact_pins,
|
||||
lockfile_check: _,
|
||||
manifest_spec_bumps,
|
||||
prompt_eligibility_override,
|
||||
@@ -111,6 +112,8 @@ where
|
||||
pnpmfile_hook_override,
|
||||
workspace_projects_override,
|
||||
} = self;
|
||||
let record_artifact_pins =
|
||||
config.lockfile && save_lockfile && (record_artifact_pins || !frozen_lockfile);
|
||||
http_client.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
|
||||
http_client_arc.set_warning_handler(pnpm_reporter::emit_global_warning::<Reporter>);
|
||||
let can_prompt = prompt_eligibility_override
|
||||
@@ -1129,6 +1132,7 @@ where
|
||||
pnpmfile_hook,
|
||||
deploy_manifest_hook,
|
||||
save_lockfile,
|
||||
record_artifact_pins,
|
||||
manifest_spec_bumps,
|
||||
catalogs: &catalogs,
|
||||
prefix: &prefix,
|
||||
|
||||
@@ -1287,7 +1287,7 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
|
||||
// ~3 ms on the alotta-files fixture, and it is what gets saved
|
||||
// below anyway.
|
||||
let phase_start = std::time::Instant::now();
|
||||
let built_lockfile = build_lockfile(FreshLockfileBuildOptions {
|
||||
let mut built_lockfile = build_lockfile(FreshLockfileBuildOptions {
|
||||
config,
|
||||
importer_manifests: &importer_manifests,
|
||||
lockfile_specifier_manifests: lockfile_specifier_manifests.as_ref(),
|
||||
@@ -1475,6 +1475,7 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
|
||||
// the hoisted-linker pass below to materialize the on-disk
|
||||
// tree. `None` for the isolated linker.
|
||||
cas_paths_by_pkg_id,
|
||||
artifact_pin_records,
|
||||
} = CreateVirtualStore {
|
||||
http_client,
|
||||
config,
|
||||
@@ -1697,6 +1698,23 @@ impl<DependencyGroupList> InstallWithFreshLockfile<'_, DependencyGroupList> {
|
||||
is_hoisted.then_some(&hoisted_locations),
|
||||
);
|
||||
|
||||
if config.lockfile && save_lockfile {
|
||||
for record in artifact_pin_records {
|
||||
if let Some(snapshot) = built_lockfile
|
||||
.snapshots
|
||||
.as_mut()
|
||||
.and_then(|snapshots| snapshots.get_mut(&record.snapshot_key))
|
||||
{
|
||||
snapshot.record_artifact_pin(
|
||||
record.input_key,
|
||||
record.owner,
|
||||
record.platform_fingerprint,
|
||||
record.envelope_digest,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Saved after the build phase succeeds so a partial install can't
|
||||
// leave a lockfile pointing at slots that never landed on disk.
|
||||
let (wanted_lockfile, can_record_lockfile_verification) = if !config.lockfile {
|
||||
@@ -2180,7 +2198,7 @@ fn build_fresh_lockfile(
|
||||
config.registries_by_prefix.iter().map(|(name, url)| (name.clone(), url.clone())),
|
||||
)
|
||||
.collect();
|
||||
let lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions {
|
||||
let mut lockfile = dependencies_graph_to_lockfile(GraphToLockfileOptions {
|
||||
importers,
|
||||
graph,
|
||||
registry_options_by_url: &config.registry_options_by_url,
|
||||
@@ -2206,6 +2224,15 @@ fn build_fresh_lockfile(
|
||||
update_reuse_scopes_by_importer,
|
||||
time: merge_recorded_time(wanted_lockfile, resolved_time),
|
||||
})?;
|
||||
if let (Some(previous), Some(snapshots)) = (
|
||||
wanted_lockfile.and_then(|lockfile| lockfile.snapshots.as_ref()),
|
||||
lockfile.snapshots.as_mut(),
|
||||
) {
|
||||
for (key, snapshot) in snapshots {
|
||||
snapshot.artifact_pins =
|
||||
previous.get(key).and_then(|previous| previous.artifact_pins.clone());
|
||||
}
|
||||
}
|
||||
Ok(lockfile)
|
||||
}
|
||||
|
||||
|
||||
@@ -336,6 +336,7 @@ pub struct ResolveArtifactsOptions {
|
||||
pub ignore_scripts: bool,
|
||||
/// P-256 `SubjectPublicKeyInfo` DER bytes keyed by the envelope's key id.
|
||||
pub trusted_keys: BTreeMap<String, Vec<u8>>,
|
||||
pub pinned_envelope_digests: BTreeMap<String, String>,
|
||||
pub authorization: Option<String>,
|
||||
}
|
||||
|
||||
@@ -577,6 +578,13 @@ impl PnprClient {
|
||||
.envelope
|
||||
.digest()
|
||||
.map_err(|err| PnprClientError::Protocol(err.to_string()))?;
|
||||
if opts
|
||||
.pinned_envelope_digests
|
||||
.get(candidate.key.as_str())
|
||||
.is_some_and(|pinned| pinned != &envelope_digest)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if best.as_ref().is_none_or(|(best_rank, best_digest, _)| {
|
||||
(rank, &envelope_digest) < (*best_rank, best_digest)
|
||||
}) {
|
||||
|
||||
@@ -896,6 +896,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
|
||||
allowed_builds: HashSet::from([candidate.package.name.clone()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), untrusted_public_key)]),
|
||||
pinned_envelope_digests: BTreeMap::new(),
|
||||
authorization: Some(pnpr_auth.clone()),
|
||||
})
|
||||
.await
|
||||
@@ -912,6 +913,7 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
|
||||
allowed_builds: HashSet::from([candidate.package.name.clone()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
|
||||
pinned_envelope_digests: BTreeMap::new(),
|
||||
authorization: Some(pnpr_auth.clone()),
|
||||
})
|
||||
.await
|
||||
@@ -925,7 +927,8 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
|
||||
eligible_packages: HashSet::from([candidate.package.name.clone()]),
|
||||
allowed_builds: HashSet::from([candidate.package.name.clone()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
|
||||
pinned_envelope_digests: BTreeMap::new(),
|
||||
authorization: Some(pnpr_auth.clone()),
|
||||
})
|
||||
.await
|
||||
@@ -933,6 +936,41 @@ async fn publishes_resolves_and_verifies_an_organization_artifact() {
|
||||
let artifact = selected.get(&publish.key).expect("trusted compatible variant selected");
|
||||
assert_eq!(artifact.payload.owner, OwnerScope::organization("pnpr-client"));
|
||||
assert_eq!(artifact.envelope_digest.len(), 64);
|
||||
let pinned_digest = artifact.envelope_digest.clone();
|
||||
let pinned = client
|
||||
.resolve_artifacts(ResolveArtifactsOptions {
|
||||
candidates: vec![candidate.clone()],
|
||||
supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()],
|
||||
eligible_packages: HashSet::from([candidate.package.name.clone()]),
|
||||
allowed_builds: HashSet::from([candidate.package.name.clone()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key.clone())]),
|
||||
pinned_envelope_digests: BTreeMap::from([(
|
||||
candidate.key.clone(),
|
||||
pinned_digest.clone(),
|
||||
)]),
|
||||
authorization: Some(pnpr_auth.clone()),
|
||||
})
|
||||
.await
|
||||
.expect("an exact pin should select the artifact");
|
||||
assert_eq!(
|
||||
pinned.get(&candidate.key).map(|artifact| &artifact.envelope_digest),
|
||||
Some(&pinned_digest),
|
||||
);
|
||||
let pinned = client
|
||||
.resolve_artifacts(ResolveArtifactsOptions {
|
||||
candidates: vec![candidate.clone()],
|
||||
supported_tags: vec!["pnpm:v1:linux-x64-node22-glibc2.17".to_string()],
|
||||
eligible_packages: HashSet::from([candidate.package.name.clone()]),
|
||||
allowed_builds: HashSet::from([candidate.package.name.clone()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
|
||||
pinned_envelope_digests: BTreeMap::from([(candidate.key.clone(), "0".repeat(64))]),
|
||||
authorization: Some(pnpr_auth.clone()),
|
||||
})
|
||||
.await
|
||||
.expect("a missing pin is a cache miss");
|
||||
assert!(pinned.is_empty());
|
||||
|
||||
let bytes = client
|
||||
.download_artifact_blob(
|
||||
@@ -977,6 +1015,7 @@ async fn artifact_lookup_preserves_script_eligibility_and_allow_build_policy() {
|
||||
allowed_builds,
|
||||
ignore_scripts,
|
||||
trusted_keys: trusted_keys.clone(),
|
||||
pinned_envelope_digests: BTreeMap::new(),
|
||||
authorization: None,
|
||||
})
|
||||
.await
|
||||
@@ -1091,6 +1130,7 @@ async fn organization_artifact_existence_is_not_exposed_to_another_owner() {
|
||||
allowed_builds: HashSet::from(["native-addon".to_string()]),
|
||||
ignore_scripts: false,
|
||||
trusted_keys: BTreeMap::from([("acme-2026".to_string(), public_key)]),
|
||||
pinned_envelope_digests: BTreeMap::new(),
|
||||
authorization: Some(pnpr_auth),
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -146,6 +146,7 @@ export type InstallDepsOptions = Pick<Config,
|
||||
}
|
||||
allowNew?: boolean
|
||||
forceFullResolution?: boolean
|
||||
refreshArtifactPins?: boolean
|
||||
frozenLockfileIfExists?: boolean
|
||||
include?: IncludedDependencies
|
||||
includeDirect?: IncludedDependencies
|
||||
@@ -182,7 +183,7 @@ export type InstallDepsOptions = Pick<Config,
|
||||
* subcommand — see `runPacquet.ts`'s `noRuntime` opt.
|
||||
*/
|
||||
isInstallCommand?: boolean
|
||||
} & Partial<Pick<Config, 'dryRun' | 'pnpmHomeDir' | 'strictDepBuilds' | 'useLockfile' | 'useGitBranchLockfile' | 'mergeGitBranchLockfiles'>>
|
||||
} & Partial<Pick<Config, 'dryRun' | 'frozenLockfile' | 'pnpmHomeDir' | 'preferFrozenLockfile' | 'strictDepBuilds' | 'useLockfile' | 'useGitBranchLockfile' | 'mergeGitBranchLockfiles'>>
|
||||
|
||||
export async function installDeps (
|
||||
opts: InstallDepsOptions,
|
||||
|
||||
@@ -85,6 +85,7 @@ export function rcOptionsTypes (): Record<string, unknown> {
|
||||
export function cliOptionsTypes (): Record<string, unknown> {
|
||||
return {
|
||||
...rcOptionsTypes(),
|
||||
'build-artifacts': Boolean,
|
||||
changeset: Boolean,
|
||||
'include-github-actions': Boolean,
|
||||
interactive: Boolean,
|
||||
@@ -140,6 +141,10 @@ For options that may be used with `-r`, see "pnpm help recursive"',
|
||||
description: 'Refresh registry revisions without changing package versions',
|
||||
name: '--patches',
|
||||
},
|
||||
{
|
||||
description: 'Update remote build artifacts without changing package versions',
|
||||
name: '--build-artifacts',
|
||||
},
|
||||
{
|
||||
description: 'Update packages only in "dependencies" and "optionalDependencies"',
|
||||
name: '--prod',
|
||||
@@ -190,6 +195,7 @@ dependencies is not found inside the workspace',
|
||||
}
|
||||
|
||||
export type UpdateCommandOptions = InstallCommandOptions & {
|
||||
buildArtifacts?: boolean
|
||||
changeset?: boolean
|
||||
include?: IncludedDependencies
|
||||
includeGithubActions?: boolean
|
||||
@@ -205,6 +211,7 @@ export async function handler (
|
||||
commands?: CommandHandlerMap
|
||||
): Promise<string | undefined> {
|
||||
assertPatchesOptions(params, opts)
|
||||
assertBuildArtifactsOptions(params, opts)
|
||||
if (opts.global) {
|
||||
if (!opts.bin) {
|
||||
throw new PnpmError('NO_GLOBAL_BIN_DIR', 'Unable to find the global bin directory', {
|
||||
@@ -418,8 +425,9 @@ async function update (
|
||||
rebuildHandler?: CommandHandler
|
||||
): Promise<void> {
|
||||
assertPatchesOptions(dependencies, opts)
|
||||
assertBuildArtifactsOptions(dependencies, opts)
|
||||
const includeDirect = makeIncludeDependenciesFromCLI(opts.cliOptions)
|
||||
const updateActions = shouldUpdateGitHubActions(opts, includeDirect)
|
||||
const updateActions = !opts.buildArtifacts && shouldUpdateGitHubActions(opts, includeDirect)
|
||||
if (opts.latest) {
|
||||
const dependenciesWithTags = dependencies.filter((name) =>
|
||||
(!updateActions || !isGitHubActionSelector(name)) && parseUpdateParam(name).versionSpec != null)
|
||||
@@ -447,7 +455,7 @@ async function update (
|
||||
} else if ((packageDependencies.length > 0) && depth > 0 && !opts.latest) {
|
||||
updateMatching = createUpdateMatching(packageDependencies.flatMap(expandUpdateSelectorsForMatching))
|
||||
}
|
||||
const generateChangeset = opts.changeset ?? opts.updateConfig?.changeset ?? false
|
||||
const generateChangeset = !opts.buildArtifacts && (opts.changeset ?? opts.updateConfig?.changeset ?? false)
|
||||
const changesetContext = generateChangeset ? await captureUpdateChangesetContext(opts) : undefined
|
||||
if (dependencies.length === 0 || packageDependencies.length > 0) {
|
||||
await installDeps({
|
||||
@@ -458,11 +466,12 @@ async function update (
|
||||
ignoreCurrentSpecifiers: false,
|
||||
include,
|
||||
includeDirect,
|
||||
update: true,
|
||||
updatePatches: opts.patches,
|
||||
refreshArtifactPins: opts.buildArtifacts,
|
||||
update: !opts.buildArtifacts,
|
||||
updatePatches: opts.buildArtifacts ? false : opts.patches,
|
||||
updateToLatest: opts.latest,
|
||||
updateMatching,
|
||||
updatePackageManifest: opts.patches ? false : opts.save !== false,
|
||||
updatePackageManifest: opts.buildArtifacts || opts.patches ? false : opts.save !== false,
|
||||
resolutionMode: opts.save === false ? 'highest' : opts.resolutionMode,
|
||||
// `--dry-run` is an `install`-only preview; never let a config-level
|
||||
// `dry-run` turn `update` into a no-op check.
|
||||
@@ -488,6 +497,29 @@ function assertPatchesOptions (dependencies: string[], opts: UpdateCommandOption
|
||||
}
|
||||
}
|
||||
|
||||
function assertBuildArtifactsOptions (dependencies: string[], opts: UpdateCommandOptions): void {
|
||||
if (!opts.buildArtifacts) return
|
||||
if (dependencies.length > 0 || opts.latest || opts.interactive || opts.global || opts.patches) {
|
||||
throw new PnpmError('BUILD_ARTIFACTS_WITH_SELECTOR', '--build-artifacts cannot be combined with package selectors, --latest, --interactive, --global, or --patches')
|
||||
}
|
||||
if (opts.useLockfile === false) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE',
|
||||
'Cannot update build artifacts when lockfile is set to false')
|
||||
}
|
||||
if (opts.lockfileOnly) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY',
|
||||
'Cannot update build artifacts without downloading and verifying them')
|
||||
}
|
||||
if (opts.cliOptions.frozenLockfile === true) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE',
|
||||
'Cannot update build artifacts with a frozen lockfile')
|
||||
}
|
||||
opts.frozenLockfile = false
|
||||
opts.frozenLockfileIfExists = false
|
||||
opts.preferFrozenLockfile = false
|
||||
opts.optimisticRepeatInstall = false
|
||||
}
|
||||
|
||||
function shouldUpdateGitHubActions (opts: UpdateCommandOptions, include: IncludedDependencies): boolean {
|
||||
return include.devDependencies &&
|
||||
opts.save !== false &&
|
||||
|
||||
@@ -3,13 +3,78 @@ import path from 'node:path'
|
||||
import { beforeAll, describe, expect, it, test } from '@jest/globals'
|
||||
import type { PnpmError } from '@pnpm/error'
|
||||
import { install, update } from '@pnpm/installing.commands'
|
||||
import { readWantedLockfile, writeWantedLockfile } from '@pnpm/lockfile.fs'
|
||||
import { prepare, preparePackages } from '@pnpm/prepare'
|
||||
import { addDistTag } from '@pnpm/testing.registry-mock'
|
||||
import type { ProjectManifest } from '@pnpm/types'
|
||||
import type { ProjectId, ProjectManifest } from '@pnpm/types'
|
||||
import { loadJsonFileSync } from 'load-json-file'
|
||||
|
||||
import { DEFAULT_OPTS } from '../utils/index.js'
|
||||
|
||||
test('update --build-artifacts clears existing pins without updating package versions', async () => {
|
||||
prepare({ dependencies: { 'is-positive': '1.0.0' } })
|
||||
await install.handler({ ...DEFAULT_OPTS, dir: process.cwd() })
|
||||
const lockfile = (await readWantedLockfile('.', { ignoreIncompatible: false }))!
|
||||
const dependencyResolution = lockfile.importers['.' as ProjectId].dependencies?.['is-positive']
|
||||
const snapshot = Object.values(lockfile.packages ?? {})[0]
|
||||
snapshot.artifactPins = {
|
||||
'dependency-side-effects:v1:deps=old': {
|
||||
'organization:acme': { platform: '0'.repeat(64) },
|
||||
},
|
||||
}
|
||||
await writeWantedLockfile('.', lockfile)
|
||||
|
||||
await update.handler({
|
||||
...DEFAULT_OPTS,
|
||||
buildArtifacts: true,
|
||||
dir: process.cwd(),
|
||||
})
|
||||
|
||||
const refreshed = (await readWantedLockfile('.', { ignoreIncompatible: false }))!
|
||||
expect(Object.values(refreshed.packages ?? {}).every(pkg => pkg.artifactPins == null)).toBe(true)
|
||||
expect(refreshed.importers['.' as ProjectId].dependencies?.['is-positive']).toStrictEqual(dependencyResolution)
|
||||
})
|
||||
|
||||
test('update --build-artifacts rejects a frozen lockfile', async () => {
|
||||
prepare({})
|
||||
await expect(update.handler({
|
||||
...DEFAULT_OPTS,
|
||||
buildArtifacts: true,
|
||||
cliOptions: { frozenLockfile: true },
|
||||
dir: process.cwd(),
|
||||
frozenLockfile: true,
|
||||
})).rejects.toMatchObject({ code: 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_FROZEN_LOCKFILE' })
|
||||
})
|
||||
|
||||
test.each([
|
||||
['a disabled lockfile', { useLockfile: false }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_NO_LOCKFILE'],
|
||||
['a lockfile-only update', { lockfileOnly: true }, 'ERR_PNPM_CONFIG_CONFLICT_BUILD_ARTIFACTS_WITH_LOCKFILE_ONLY'],
|
||||
] as const)('update --build-artifacts rejects %s', async (_description, conflictingOptions, code) => {
|
||||
prepare({})
|
||||
await expect(update.handler({
|
||||
...DEFAULT_OPTS,
|
||||
...conflictingOptions,
|
||||
buildArtifacts: true,
|
||||
dir: process.cwd(),
|
||||
})).rejects.toMatchObject({ code })
|
||||
})
|
||||
|
||||
test.each([
|
||||
{ dependencies: ['is-positive'], options: { buildArtifacts: true } },
|
||||
{ dependencies: [], options: { buildArtifacts: true, latest: true } },
|
||||
{ dependencies: [], options: { buildArtifacts: true, interactive: true } },
|
||||
{ dependencies: [], options: { buildArtifacts: true, global: true } },
|
||||
{ dependencies: [], options: { buildArtifacts: true, patches: true } },
|
||||
])('update --build-artifacts rejects other update modes', async ({ dependencies, options }) => {
|
||||
await expect(update.handler({
|
||||
...DEFAULT_OPTS,
|
||||
...options,
|
||||
dir: process.cwd(),
|
||||
}, dependencies)).rejects.toMatchObject({
|
||||
code: 'ERR_PNPM_BUILD_ARTIFACTS_WITH_SELECTOR',
|
||||
})
|
||||
})
|
||||
|
||||
test.each([
|
||||
{ dependencies: ['is-positive'], options: { patches: true } },
|
||||
{ dependencies: [], options: { latest: true, patches: true } },
|
||||
|
||||
@@ -44,6 +44,7 @@ export interface StrictInstallOptions extends RegistryContext {
|
||||
forceFullResolution: boolean
|
||||
fixLockfile: boolean
|
||||
updateChecksums: boolean
|
||||
refreshArtifactPins: boolean
|
||||
dedupe: boolean
|
||||
ignoreCompatibilityDb: boolean
|
||||
ignorePackageManifest: boolean
|
||||
@@ -320,6 +321,7 @@ const defaults = (opts: InstallOptions): StrictInstallOptions => {
|
||||
lockfileDir: opts.lockfileDir ?? opts.dir ?? process.cwd(),
|
||||
lockfileOnly: false,
|
||||
updateChecksums: false,
|
||||
refreshArtifactPins: false,
|
||||
nodeVersion: opts.nodeVersion,
|
||||
nodeLinker: 'isolated',
|
||||
nodeExperimentalPackageMap: false,
|
||||
@@ -449,6 +451,18 @@ export function extendOptions (
|
||||
`Cannot generate a ${WANTED_LOCKFILE} because lockfile is set to false`)
|
||||
}
|
||||
}
|
||||
if (extendedOpts.refreshArtifactPins) {
|
||||
if (!extendedOpts.useLockfile) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_NO_LOCKFILE',
|
||||
'Cannot refresh artifact pins when lockfile is set to false')
|
||||
}
|
||||
if (extendedOpts.lockfileOnly) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_REFRESH_ARTIFACT_PINS_WITH_LOCKFILE_ONLY',
|
||||
'Cannot refresh artifact pins without downloading and verifying artifacts')
|
||||
}
|
||||
extendedOpts.force = true
|
||||
extendedOpts.preferFrozenLockfile = false
|
||||
}
|
||||
if (extendedOpts.frozenStore && extendedOpts.force) {
|
||||
throw new PnpmError('CONFIG_CONFLICT_FROZEN_STORE_WITH_FORCE',
|
||||
'Cannot use force together with frozenStore: --force re-imports packages into the store, which is opened read-only when frozenStore is enabled')
|
||||
|
||||
@@ -369,7 +369,7 @@ export async function mutateModules (
|
||||
// (pnpm remove), and complete-project revision refreshes. Mutations that
|
||||
// need other client-side update behavior still fall through to the normal
|
||||
// flow.
|
||||
if (opts.pnprServer && canUsePnprForMutations(projects, opts)) {
|
||||
if (opts.pnprServer && !opts.refreshArtifactPins && canUsePnprForMutations(projects, opts)) {
|
||||
const pnprResult = await mutateModulesViaPnpr(projects, opts)
|
||||
if (pnprResult) {
|
||||
// This path materializes packages of its own, so it verifies the
|
||||
@@ -404,6 +404,9 @@ export async function mutateModules (
|
||||
await safeReadProjectManifestOnly(opts.lockfileDir)
|
||||
|
||||
let ctx = await getContext(opts)
|
||||
if (opts.refreshArtifactPins && clearArtifactPins(ctx.wantedLockfile)) {
|
||||
ctx.wantedLockfileIsModified = true
|
||||
}
|
||||
|
||||
if (!opts.lockfileOnly && !isCheckOnlyInstall(opts) && ctx.modulesFile != null) {
|
||||
const { purged } = await validateModules(ctx.modulesFile, Object.values(ctx.projects), {
|
||||
@@ -1502,6 +1505,7 @@ Note that in CI environments, this setting is enabled by default.`,
|
||||
relinkChangedDependenciesOnly: didFastUpdateOverrides,
|
||||
wantedLockfile: maybeOpts.ignorePackageManifest ? undefined : ctx.wantedLockfile,
|
||||
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
|
||||
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !frozenLockfile,
|
||||
verifyLockfile,
|
||||
})
|
||||
if (
|
||||
@@ -2146,6 +2150,7 @@ const _installInContext: InstallFunction = async (projects, ctx, opts) => {
|
||||
virtualStoreDir: ctx.virtualStoreDir,
|
||||
virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
|
||||
wantedLockfile: newLockfile,
|
||||
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
|
||||
wantedToBeSkippedPackageIds,
|
||||
hoistWorkspacePackages: opts.hoistWorkspacePackages,
|
||||
virtualStoreOnly: opts.virtualStoreOnly,
|
||||
@@ -2617,6 +2622,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
|
||||
prunedAt: ctx.modulesFile?.prunedAt,
|
||||
wantedLockfile: result.newLockfile,
|
||||
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
|
||||
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
|
||||
hoistWorkspacePackages: opts.hoistWorkspacePackages,
|
||||
}))
|
||||
return {
|
||||
@@ -2647,6 +2653,7 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
|
||||
prunedAt: ctx.modulesFile?.prunedAt,
|
||||
wantedLockfile: result.newLockfile,
|
||||
useLockfile: opts.useLockfile && ctx.wantedLockfileIsModified,
|
||||
recordArtifactPins: opts.useLockfile && opts.saveLockfile && !opts.frozenLockfile,
|
||||
hoistWorkspacePackages: opts.hoistWorkspacePackages,
|
||||
}))
|
||||
return {
|
||||
@@ -2740,6 +2747,16 @@ const installInContext: InstallFunction = async (projects, ctx, opts) => {
|
||||
|
||||
const limitLinking = pLimit(16)
|
||||
|
||||
function clearArtifactPins (lockfile: LockfileObject): boolean {
|
||||
let changed = false
|
||||
for (const snapshot of Object.values(lockfile.packages ?? {})) {
|
||||
if (snapshot.artifactPins == null) continue
|
||||
delete snapshot.artifactPins
|
||||
changed = true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
async function linkAllBins (
|
||||
depNodes: DependenciesGraphNode[],
|
||||
depGraph: DependenciesGraph,
|
||||
@@ -3142,6 +3159,8 @@ async function installViaPnprServer (
|
||||
patchedDependencies,
|
||||
resolvedPatchedDependencies
|
||||
)
|
||||
const frozenLockfile = opts.frozenLockfile === true ||
|
||||
(opts.frozenLockfileIfExists === true && existingLockfile != null)
|
||||
|
||||
logger.info({ message: 'Resolving dependencies via the pnpr server', prefix: rootDir })
|
||||
|
||||
@@ -3193,7 +3212,7 @@ async function installViaPnprServer (
|
||||
// Lockfile reuse. Without these the server always reuse-and-updates,
|
||||
// so `--frozen-lockfile` would silently resolve and rewrite the very
|
||||
// lockfile it promises to leave alone.
|
||||
frozenLockfile: opts.frozenLockfile === true || (opts.frozenLockfileIfExists === true && existingLockfile != null),
|
||||
frozenLockfile,
|
||||
preferFrozenLockfile: opts.preferFrozenLockfile,
|
||||
updatePatches: opts.updatePatches,
|
||||
lockfile: existingLockfile ?? undefined,
|
||||
@@ -3268,6 +3287,7 @@ async function installViaPnprServer (
|
||||
patchedDependencies: patchGroups,
|
||||
skipped: new Set<DepPath>(),
|
||||
wantedLockfile: lockfile,
|
||||
recordArtifactPins: opts.useLockfile !== false && opts.saveLockfile !== false && !frozenLockfile,
|
||||
}
|
||||
const { ignoredBuilds, stats } = await materializeOrDelegate(
|
||||
opts,
|
||||
|
||||
@@ -72,6 +72,7 @@ export interface LinkPackagesOptions {
|
||||
sideEffectsCacheRead: boolean
|
||||
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
|
||||
pnprServer?: string
|
||||
recordArtifactPins: boolean
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
symlink: boolean
|
||||
skipped: Set<DepPath>
|
||||
@@ -171,6 +172,8 @@ export async function linkPackages (projects: ImporterToUpdate[], depGraph: Depe
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
remoteSideEffectsCache: opts.remoteSideEffectsCache,
|
||||
pnprServer: opts.pnprServer,
|
||||
artifactPinsLockfile: opts.wantedLockfile,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
configByUri: opts.configByUri,
|
||||
symlink: opts.symlink,
|
||||
skipped: opts.skipped,
|
||||
@@ -346,6 +349,8 @@ interface LinkNewPackagesOptions {
|
||||
sideEffectsCacheRead: boolean
|
||||
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
|
||||
pnprServer?: string
|
||||
artifactPinsLockfile: LockfileObject
|
||||
recordArtifactPins: boolean
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
symlink: boolean
|
||||
skipped: Set<DepPath>
|
||||
@@ -466,6 +471,8 @@ async function linkNewPackages (
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
remoteSideEffectsCache: opts.remoteSideEffectsCache,
|
||||
pnprServer: opts.pnprServer,
|
||||
artifactPinsLockfile: opts.artifactPinsLockfile,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
configByUri: opts.configByUri,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
}),
|
||||
@@ -526,6 +533,8 @@ async function linkAllPkgs (
|
||||
sideEffectsCacheRead: boolean
|
||||
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
|
||||
pnprServer?: string
|
||||
artifactPinsLockfile: LockfileObject
|
||||
recordArtifactPins: boolean
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
supportedArchitectures?: SupportedArchitectures
|
||||
}
|
||||
@@ -537,12 +546,14 @@ async function linkAllPkgs (
|
||||
const nodeVersion = findRuntimeNodeVersion(Object.keys(opts.depGraph))
|
||||
const restorer = createRemoteSideEffectsRestorer({
|
||||
allowBuild: opts.allowBuild,
|
||||
artifactPinsLockfile: opts.artifactPinsLockfile,
|
||||
configByUri: opts.configByUri,
|
||||
depsGraph: opts.depGraph,
|
||||
depsStateCache: opts.depsStateCache,
|
||||
ignoreScripts: opts.ignoreScripts,
|
||||
nodeVersion,
|
||||
pnprServer: opts.pnprServer,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
settings: opts.remoteSideEffectsCache,
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
storeController,
|
||||
|
||||
@@ -109,6 +109,9 @@ function toLockfileDependency (
|
||||
const result = {
|
||||
resolution: lockfileResolution,
|
||||
} as PackageSnapshot
|
||||
if (opts.prevSnapshot?.artifactPins != null) {
|
||||
result.artifactPins = opts.prevSnapshot.artifactPins
|
||||
}
|
||||
// A registry-qualified dep path (`<name>@<registryName>:<version>`) already
|
||||
// carries a parseable semver, so the explicit version field written for
|
||||
// other `:`-containing dep paths would be redundant.
|
||||
|
||||
@@ -111,6 +111,7 @@ export interface Project {
|
||||
}
|
||||
|
||||
export interface HeadlessOptions extends RegistryContext {
|
||||
recordArtifactPins?: boolean
|
||||
projectDependencies?: Map<ProjectRootDir, ProjectRootDir[]>
|
||||
allowBuilds?: Record<string, boolean | string>
|
||||
autoInstallPeers?: boolean
|
||||
@@ -237,6 +238,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
if (wantedLockfile == null) {
|
||||
throw new Error(`Headless installation requires a ${WANTED_LOCKFILE} file`)
|
||||
}
|
||||
let artifactPinsChanged = false
|
||||
|
||||
const depsStateCache: DepsStateCache = {}
|
||||
// `modulesDir` is conventionally a path relative to `lockfileDir`, but
|
||||
@@ -448,6 +450,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
if (!skipPostImportLinking) {
|
||||
await linkHoistedModules(opts.storeController, graph, prevGraph, hierarchy, {
|
||||
allowBuild,
|
||||
artifactPinsLockfile: wantedLockfile,
|
||||
depsStateCache,
|
||||
disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
|
||||
force: opts.force,
|
||||
@@ -457,6 +460,10 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
remoteSideEffectsCache: opts.remoteSideEffectsCache,
|
||||
pnprServer: opts.pnprServer,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
onArtifactPinsChanged: () => {
|
||||
artifactPinsChanged = true
|
||||
},
|
||||
configByUri: opts.configByUri,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
})
|
||||
@@ -490,6 +497,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
}),
|
||||
linkAllPkgs(opts.storeController, depNodes, {
|
||||
allowBuild,
|
||||
artifactPinsLockfile: wantedLockfile,
|
||||
force: opts.force,
|
||||
disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
|
||||
depGraph: graph,
|
||||
@@ -500,6 +508,10 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
remoteSideEffectsCache: opts.remoteSideEffectsCache,
|
||||
pnprServer: opts.pnprServer,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
onArtifactPinsChanged: () => {
|
||||
artifactPinsChanged = true
|
||||
},
|
||||
configByUri: opts.configByUri,
|
||||
storeDir: opts.storeDir,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
@@ -788,7 +800,7 @@ export async function headlessInstall (opts: HeadlessOptions): Promise<Installat
|
||||
virtualStoreOnly: opts.virtualStoreOnly,
|
||||
})
|
||||
const currentLockfileDir = path.join(rootModulesDir, '.pnpm')
|
||||
if (opts.useLockfile) {
|
||||
if (opts.useLockfile || artifactPinsChanged) {
|
||||
// We need to write the wanted lockfile as well.
|
||||
// Even though it will only be changed if the workspace will have new projects with no dependencies.
|
||||
await writeLockfiles({
|
||||
@@ -1023,6 +1035,7 @@ async function linkAllPkgs (
|
||||
depNodes: DependenciesGraphNode[],
|
||||
opts: {
|
||||
allowBuild?: AllowBuild
|
||||
artifactPinsLockfile?: LockfileObject
|
||||
depGraph: DependenciesGraph
|
||||
depsStateCache: DepsStateCache
|
||||
disableRelinkLocalDirDeps?: boolean
|
||||
@@ -1033,6 +1046,8 @@ async function linkAllPkgs (
|
||||
sideEffectsCacheRead: boolean
|
||||
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
|
||||
pnprServer?: string
|
||||
recordArtifactPins?: boolean
|
||||
onArtifactPinsChanged?: () => void
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
storeDir: string
|
||||
supportedArchitectures?: SupportedArchitectures
|
||||
@@ -1056,16 +1071,19 @@ async function linkAllPkgs (
|
||||
const nodeVersion = findRuntimeNodeVersion(depNodes.map((node) => node.depPath))
|
||||
const restorer = createRemoteSideEffectsRestorer({
|
||||
allowBuild: opts.allowBuild,
|
||||
artifactPinsLockfile: opts.artifactPinsLockfile,
|
||||
configByUri: opts.configByUri,
|
||||
depsGraph: opts.depGraph,
|
||||
depsStateCache: opts.depsStateCache,
|
||||
ignoreScripts: opts.ignoreScripts,
|
||||
nodeVersion,
|
||||
pnprServer: opts.pnprServer,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
settings: opts.remoteSideEffectsCache,
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
storeController,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
onArtifactPinsChanged: opts.onArtifactPinsChanged,
|
||||
warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }),
|
||||
})
|
||||
await Promise.all(
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
DepHierarchy,
|
||||
} from '@pnpm/deps.graph-builder'
|
||||
import { calcDepState, type DepsStateCache, findRuntimeNodeVersion } from '@pnpm/deps.graph-hasher'
|
||||
import type { LockfileObject } from '@pnpm/lockfile.fs'
|
||||
import { logger } from '@pnpm/logger'
|
||||
import { createRemoteSideEffectsRestorer, type RemoteSideEffectsRestorer } from '@pnpm/pnpr.client'
|
||||
import type {
|
||||
@@ -31,6 +32,7 @@ export async function linkHoistedModules (
|
||||
hierarchy: DepHierarchy,
|
||||
opts: {
|
||||
allowBuild?: AllowBuild
|
||||
artifactPinsLockfile?: LockfileObject
|
||||
depsStateCache: DepsStateCache
|
||||
disableRelinkLocalDirDeps?: boolean
|
||||
force: boolean
|
||||
@@ -40,6 +42,8 @@ export async function linkHoistedModules (
|
||||
sideEffectsCacheRead: boolean
|
||||
remoteSideEffectsCache?: RemoteSideEffectsCacheSettings
|
||||
pnprServer?: string
|
||||
recordArtifactPins?: boolean
|
||||
onArtifactPinsChanged?: () => void
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
supportedArchitectures?: SupportedArchitectures
|
||||
}
|
||||
@@ -67,16 +71,19 @@ export async function linkHoistedModules (
|
||||
)
|
||||
const restorer = createRemoteSideEffectsRestorer({
|
||||
allowBuild: opts.allowBuild,
|
||||
artifactPinsLockfile: opts.artifactPinsLockfile,
|
||||
configByUri: opts.configByUri,
|
||||
depsGraph: graph,
|
||||
depsStateCache: opts.depsStateCache,
|
||||
ignoreScripts: opts.ignoreScripts,
|
||||
nodeVersion,
|
||||
pnprServer: opts.pnprServer,
|
||||
recordArtifactPins: opts.recordArtifactPins,
|
||||
settings: opts.remoteSideEffectsCache,
|
||||
sideEffectsCacheRead: opts.sideEffectsCacheRead,
|
||||
storeController,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
onArtifactPinsChanged: opts.onArtifactPinsChanged,
|
||||
warn: (message) => logger.warn({ message, prefix: opts.lockfileDir }),
|
||||
})
|
||||
await Promise.all(
|
||||
|
||||
@@ -21,6 +21,7 @@ export function convertToLockfileFile (lockfile: LockfileObject): LockfileFile {
|
||||
const snapshots: Record<string, LockfilePackageSnapshot> = {}
|
||||
for (const [depPath, pkg] of Object.entries(lockfile.packages ?? {})) {
|
||||
snapshots[depPath] = pick([
|
||||
'artifactPins',
|
||||
'dependencies',
|
||||
'optionalDependencies',
|
||||
'transitivePeerDependencies',
|
||||
|
||||
@@ -25,9 +25,10 @@ const ORDERED_KEYS = {
|
||||
dependencies: 16,
|
||||
optionalDependencies: 17,
|
||||
|
||||
transitivePeerDependencies: 18,
|
||||
dev: 19,
|
||||
optional: 20,
|
||||
artifactPins: 18,
|
||||
transitivePeerDependencies: 19,
|
||||
dev: 20,
|
||||
optional: 21,
|
||||
}
|
||||
|
||||
type RootKey = keyof LockfileFile
|
||||
|
||||
@@ -2,7 +2,31 @@ import { expect, test } from '@jest/globals'
|
||||
import { LOCKFILE_VERSION } from '@pnpm/constants'
|
||||
import type { DepPath, ProjectId } from '@pnpm/types'
|
||||
|
||||
import { convertToLockfileFile } from '../lib/lockfileFormatConverters.js'
|
||||
import { convertToLockfileFile, convertToLockfileObject } from '../lib/lockfileFormatConverters.js'
|
||||
|
||||
test('artifact pins round-trip in package snapshots', () => {
|
||||
const lockfile = {
|
||||
lockfileVersion: LOCKFILE_VERSION,
|
||||
importers: {},
|
||||
packages: {
|
||||
['native-addon@1.0.0' as DepPath]: {
|
||||
artifactPins: {
|
||||
'dependency-side-effects:v1:deps=abc': {
|
||||
'organization:acme': { platform: 'digest' },
|
||||
},
|
||||
},
|
||||
resolution: { integrity: 'sha512-AAAA' },
|
||||
},
|
||||
},
|
||||
}
|
||||
const serialized = convertToLockfileFile(lockfile)
|
||||
expect(serialized.snapshots?.['native-addon@1.0.0'].artifactPins).toEqual(
|
||||
lockfile.packages['native-addon@1.0.0'].artifactPins
|
||||
)
|
||||
expect(convertToLockfileObject(serialized).packages?.['native-addon@1.0.0' as DepPath].artifactPins).toEqual(
|
||||
lockfile.packages['native-addon@1.0.0'].artifactPins
|
||||
)
|
||||
})
|
||||
|
||||
test('empty overrides are removed during lockfile normalization', () => {
|
||||
expect(convertToLockfileFile({
|
||||
|
||||
@@ -32,12 +32,15 @@ export interface LockfileObject extends LockfileBase {
|
||||
}
|
||||
|
||||
export interface LockfilePackageSnapshot {
|
||||
artifactPins?: ArtifactPins
|
||||
optional?: true
|
||||
dependencies?: ResolvedDependencies
|
||||
optionalDependencies?: ResolvedDependencies
|
||||
transitivePeerDependencies?: string[]
|
||||
}
|
||||
|
||||
export type ArtifactPins = Record<string, Record<string, Record<string, string>>>
|
||||
|
||||
export interface LockfilePackageInfo {
|
||||
id?: string
|
||||
patched?: true
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'node:fs/promises'
|
||||
import util from 'node:util'
|
||||
|
||||
import { calcDepState, calcDepStateInputKey, type DepsGraph, type DepsStateCache } from '@pnpm/deps.graph-hasher'
|
||||
import type { LockfileResolution } from '@pnpm/lockfile.types'
|
||||
import type { ArtifactPins, LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
|
||||
import { createGetAuthHeaderByURI } from '@pnpm/network.auth-header'
|
||||
import type { PackageFilesResponse, StoreController, UploadPkgToStoreResult } from '@pnpm/store.controller-types'
|
||||
import type { AllowBuild, DepPath, RegistryConfig, RemoteSideEffectsCacheSettings, SupportedArchitectures } from '@pnpm/types'
|
||||
@@ -19,9 +19,12 @@ import {
|
||||
downloadSharedArtifactBlob,
|
||||
linuxGlibcCompatibilityTag,
|
||||
linuxGlibcSupportedTags,
|
||||
ownerNamespace,
|
||||
platformFingerprint,
|
||||
pnprSupportsSharedSideEffects,
|
||||
publishSharedSideEffects,
|
||||
resolveSharedSideEffects,
|
||||
type VerifiedArtifact,
|
||||
} from './sharedSideEffects.js'
|
||||
|
||||
export interface RemoteSideEffectsInstallNode<T extends string> {
|
||||
@@ -36,16 +39,19 @@ export interface RemoteSideEffectsInstallNode<T extends string> {
|
||||
|
||||
export interface RemoteSideEffectsRestorerOptions<T extends string> {
|
||||
allowBuild?: AllowBuild
|
||||
artifactPinsLockfile?: LockfileObject
|
||||
configByUri: Record<string, RegistryConfig>
|
||||
depsGraph: DepsGraph<T>
|
||||
depsStateCache: DepsStateCache
|
||||
ignoreScripts: boolean
|
||||
nodeVersion?: string
|
||||
pnprServer?: string
|
||||
recordArtifactPins?: boolean
|
||||
settings?: RemoteSideEffectsCacheSettings
|
||||
sideEffectsCacheRead: boolean
|
||||
storeController: StoreController
|
||||
supportedArchitectures?: SupportedArchitectures
|
||||
onArtifactPinsChanged?: () => void
|
||||
warn?: (message: string) => void
|
||||
}
|
||||
|
||||
@@ -83,6 +89,7 @@ const MAX_LOOKUP_BATCH = 512
|
||||
interface RestoredArtifact {
|
||||
added: Map<string, string>
|
||||
deleted: string[]
|
||||
envelopeDigest: string
|
||||
}
|
||||
|
||||
interface QueuedLookup {
|
||||
@@ -111,6 +118,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
if (platform == null || pnprServer == null || settings == null || organization == null) return undefined
|
||||
const registryUrl = pnprServer
|
||||
const ownerName = organization
|
||||
const owner = { type: 'organization', name: ownerName } as const
|
||||
let supportedTags: string[]
|
||||
try {
|
||||
supportedTags = linuxGlibcSupportedTags(platform)
|
||||
@@ -119,6 +127,10 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
return undefined
|
||||
}
|
||||
const trustedKeys = settings.trustedKeys ?? {}
|
||||
const ownerKey = ownerNamespace(owner)
|
||||
const fingerprint = platformFingerprint(supportedTags)
|
||||
const pinnedEnvelopeDigests = new Map<string, string>()
|
||||
const pinCollisions = new Set<string>()
|
||||
const eligiblePackages = new Set(settings.packages)
|
||||
const authorization = createGetAuthHeaderByURI(opts.configByUri)(registryUrl)
|
||||
const artifactLimit = pLimit(4)
|
||||
@@ -151,6 +163,21 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
patchFileHash: node.patchFileHash,
|
||||
supportedArchitectures: opts.supportedArchitectures,
|
||||
})
|
||||
const pinnedEnvelopeDigest = opts.artifactPinsLockfile?.packages?.[node.depPath]
|
||||
.artifactPins?.[inputKey]?.[ownerKey]?.[fingerprint]
|
||||
if (pinnedEnvelopeDigest != null) {
|
||||
const previous = pinnedEnvelopeDigests.get(inputKey)
|
||||
if (previous == null) {
|
||||
pinnedEnvelopeDigests.set(inputKey, pinnedEnvelopeDigest)
|
||||
} else if (previous !== pinnedEnvelopeDigest) {
|
||||
pinnedEnvelopeDigests.delete(inputKey)
|
||||
pinCollisions.add(inputKey)
|
||||
}
|
||||
}
|
||||
if (pinCollisions.has(inputKey)) {
|
||||
opts.warn?.(`Conflicting remote side-effects pins for ${node.name}@${node.version}; building locally`)
|
||||
return undefined
|
||||
}
|
||||
if (collisions.has(inputKey)) return undefined
|
||||
const identity = `${node.name}\0${node.version}\0${sourceIntegrity}`
|
||||
const knownIdentity = identityByInputKey.get(inputKey)
|
||||
@@ -179,14 +206,20 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
key: inputKey,
|
||||
package: { name: node.name, version: node.version },
|
||||
sourceIntegrity,
|
||||
owner: { type: 'organization', name: ownerName },
|
||||
owner,
|
||||
})
|
||||
lookups.set(inputKey, lookup)
|
||||
}
|
||||
const artifact = await lookup
|
||||
if (artifact == null) return undefined
|
||||
if (artifact == null) {
|
||||
if (pinnedEnvelopeDigests.has(inputKey)) {
|
||||
opts.warn?.(`Pinned remote side-effects artifact for ${node.name}@${node.version} is unavailable; building locally`)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
recordArtifactPin(node.depPath, inputKey, artifact.envelopeDigest)
|
||||
node.files.sideEffectsMaps ??= new Map()
|
||||
node.files.sideEffectsMaps.set(localCacheKey, artifact)
|
||||
node.files.sideEffectsMaps.set(localCacheKey, { added: artifact.added, deleted: artifact.deleted })
|
||||
return localCacheKey
|
||||
}
|
||||
|
||||
@@ -216,6 +249,13 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
}
|
||||
|
||||
async function lookupBatch (batch: QueuedLookup[]): Promise<void> {
|
||||
const eligibleBatch = batch.filter(({ candidate, resolve }) => {
|
||||
if (!collisions.has(candidate.key) && !pinCollisions.has(candidate.key)) return true
|
||||
resolve(undefined)
|
||||
return false
|
||||
})
|
||||
if (eligibleBatch.length === 0) return
|
||||
const batchPinnedEnvelopeDigests = new Map(pinnedEnvelopeDigests)
|
||||
supported ??= (async () => {
|
||||
try {
|
||||
return await pnprSupportsSharedSideEffects({ registryUrl, authorization })
|
||||
@@ -225,7 +265,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
}
|
||||
})()
|
||||
if (!await supported) {
|
||||
for (const { resolve } of batch) resolve(undefined)
|
||||
for (const { resolve } of eligibleBatch) resolve(undefined)
|
||||
return
|
||||
}
|
||||
let resolved
|
||||
@@ -233,21 +273,26 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
resolved = await resolveSharedSideEffects({
|
||||
registryUrl,
|
||||
authorization,
|
||||
candidates: batch.map(({ candidate }) => candidate),
|
||||
candidates: eligibleBatch.map(({ candidate }) => candidate),
|
||||
supportedTags,
|
||||
policy: {
|
||||
ignoreScripts: false,
|
||||
eligiblePackages,
|
||||
allowedBuilds: new Set(batch.map(({ candidate }) => candidate.package.name)),
|
||||
allowedBuilds: new Set(eligibleBatch.map(({ candidate }) => candidate.package.name)),
|
||||
},
|
||||
trustedKeys,
|
||||
pinnedEnvelopeDigests: batchPinnedEnvelopeDigests,
|
||||
})
|
||||
} catch (err: unknown) {
|
||||
opts.warn?.(`Remote side-effects cache lookup failed: ${errorMessage(err)}`)
|
||||
for (const { resolve } of batch) resolve(undefined)
|
||||
for (const { resolve } of eligibleBatch) resolve(undefined)
|
||||
return
|
||||
}
|
||||
await Promise.all(batch.map(async ({ candidate, resolve }) => {
|
||||
await Promise.all(eligibleBatch.map(async ({ candidate, resolve }) => {
|
||||
if (collisions.has(candidate.key) || pinCollisions.has(candidate.key)) {
|
||||
resolve(undefined)
|
||||
return
|
||||
}
|
||||
const artifact = resolved.get(candidate.key)
|
||||
if (artifact == null) {
|
||||
resolve(undefined)
|
||||
@@ -258,7 +303,7 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
}
|
||||
|
||||
async function hydrate (
|
||||
artifact: { payload: ArtifactPayload },
|
||||
artifact: VerifiedArtifact,
|
||||
candidate: ArtifactCandidate
|
||||
): Promise<RestoredArtifact | undefined> {
|
||||
try {
|
||||
@@ -295,12 +340,33 @@ export function createRemoteSideEffectsRestorer<T extends string> (
|
||||
throw err
|
||||
}
|
||||
})))
|
||||
return { added, deleted: artifact.payload.manifest.deleted }
|
||||
return { added, deleted: artifact.payload.manifest.deleted, envelopeDigest: artifact.envelopeDigest }
|
||||
} catch (err: unknown) {
|
||||
opts.warn?.(`Remote side-effects artifact for ${candidate.package.name}@${candidate.package.version} was rejected: ${errorMessage(err)}`)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function recordArtifactPin (depPath: DepPath, inputKey: string, envelopeDigest: string): void {
|
||||
if (opts.recordArtifactPins !== true) return
|
||||
const snapshot = opts.artifactPinsLockfile?.packages?.[depPath]
|
||||
if (snapshot == null) return
|
||||
const previous = snapshot.artifactPins?.[inputKey]
|
||||
if (previous?.[ownerKey]?.[fingerprint] === envelopeDigest) return
|
||||
const artifactPins: ArtifactPins = {
|
||||
...snapshot.artifactPins,
|
||||
[inputKey]: {
|
||||
...previous,
|
||||
[ownerKey]: {
|
||||
...previous?.[ownerKey],
|
||||
[fingerprint]: envelopeDigest,
|
||||
},
|
||||
},
|
||||
}
|
||||
snapshot.artifactPins = artifactPins
|
||||
pinnedEnvelopeDigests.set(inputKey, envelopeDigest)
|
||||
opts.onArtifactPinsChanged?.()
|
||||
}
|
||||
}
|
||||
|
||||
export interface PublishBuiltSharedSideEffectsOptions<T extends string> {
|
||||
|
||||
@@ -130,6 +130,7 @@ export interface ResolveSharedSideEffectsOptions {
|
||||
}
|
||||
/** Base64-encoded P-256 SubjectPublicKeyInfo DER, keyed by key id. */
|
||||
trustedKeys: Record<string, string>
|
||||
pinnedEnvelopeDigests?: ReadonlyMap<string, string>
|
||||
}
|
||||
|
||||
interface ArtifactVariant {
|
||||
@@ -266,6 +267,8 @@ export async function resolveSharedSideEffects (
|
||||
const rank = compatibilityRank(payload.compatibility, opts.supportedTags)
|
||||
if (rank == null) continue
|
||||
const digest = signedArtifactEnvelopeDigest(variant.envelope)
|
||||
const pinnedDigest = opts.pinnedEnvelopeDigests?.get(candidate.key)
|
||||
if (pinnedDigest != null && digest !== pinnedDigest) continue
|
||||
if (
|
||||
best == null ||
|
||||
rank < best.rank ||
|
||||
@@ -286,6 +289,12 @@ export async function resolveSharedSideEffects (
|
||||
return selected
|
||||
}
|
||||
|
||||
export function ownerNamespace (owner: OwnerScope): string {
|
||||
return owner.type === 'organization'
|
||||
? `organization:${owner.name}`
|
||||
: `publisher:${owner.package}`
|
||||
}
|
||||
|
||||
export function verifySignedArtifactEnvelope (
|
||||
envelope: SignedArtifactEnvelope,
|
||||
publicKeySpki: string
|
||||
|
||||
@@ -7,7 +7,7 @@ import { setTimeout as delay } from 'node:timers/promises'
|
||||
|
||||
import { describe, expect, test } from '@jest/globals'
|
||||
import type { DepsGraph } from '@pnpm/deps.graph-hasher'
|
||||
import type { LockfileResolution } from '@pnpm/lockfile.types'
|
||||
import type { LockfileObject, LockfileResolution } from '@pnpm/lockfile.types'
|
||||
import {
|
||||
type ArtifactPayload,
|
||||
createRemoteSideEffectsRestorer,
|
||||
@@ -38,6 +38,7 @@ describe('install remote side-effects', () => {
|
||||
'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'),
|
||||
}
|
||||
const requestedPaths: string[] = []
|
||||
let heldResolve: { wait: Promise<void>, notifyStarted: () => void } | undefined
|
||||
const server = createServer((request, response) => {
|
||||
const chunks: Buffer[] = []
|
||||
request.on('data', chunk => chunks.push(Buffer.from(chunk)))
|
||||
@@ -96,8 +97,18 @@ describe('install remote side-effects', () => {
|
||||
}],
|
||||
}
|
||||
})
|
||||
response.writeHead(200, { 'content-type': 'application/json' })
|
||||
.end(JSON.stringify({ artifacts: envelopes }))
|
||||
const sendResponse = (): void => {
|
||||
response.writeHead(200, { 'content-type': 'application/json' })
|
||||
.end(JSON.stringify({ artifacts: envelopes }))
|
||||
}
|
||||
const held = heldResolve
|
||||
if (held == null) {
|
||||
sendResponse()
|
||||
} else {
|
||||
heldResolve = undefined
|
||||
held.notifyStarted()
|
||||
void held.wait.then(sendResponse)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (request.url === '/-/pnpr/v0/artifacts/blob') {
|
||||
@@ -132,15 +143,32 @@ describe('install remote side-effects', () => {
|
||||
fullPkgId: graphKey,
|
||||
},
|
||||
}
|
||||
const artifactPinsLockfile: LockfileObject = {
|
||||
lockfileVersion: '9.0',
|
||||
importers: {},
|
||||
packages: {
|
||||
[depPath]: {
|
||||
artifactPins: {
|
||||
'dependency-side-effects:v1:deps=other': {
|
||||
'organization:acme': { 'other-platform': '1'.repeat(64) },
|
||||
},
|
||||
},
|
||||
resolution: { integrity: sourceIntegrity },
|
||||
},
|
||||
},
|
||||
}
|
||||
let artifactPinsChanged = 0
|
||||
|
||||
try {
|
||||
const restorer = createRemoteSideEffectsRestorer({
|
||||
allowBuild: candidate => candidate === depPath,
|
||||
artifactPinsLockfile,
|
||||
configByUri: {},
|
||||
depsGraph,
|
||||
depsStateCache: {},
|
||||
ignoreScripts: false,
|
||||
pnprServer,
|
||||
recordArtifactPins: true,
|
||||
settings: {
|
||||
organization: 'acme',
|
||||
packages: [packageName],
|
||||
@@ -148,6 +176,9 @@ describe('install remote side-effects', () => {
|
||||
},
|
||||
sideEffectsCacheRead: false,
|
||||
storeController,
|
||||
onArtifactPinsChanged: () => {
|
||||
artifactPinsChanged++
|
||||
},
|
||||
})
|
||||
const cacheKey = await restorer?.restore({
|
||||
graphKey,
|
||||
@@ -167,6 +198,17 @@ describe('install remote side-effects', () => {
|
||||
deleted: ['src/intermediate.o'],
|
||||
})
|
||||
expect(storedFiles).toEqual([{ bytes: builtFile, mode: 0o755 }])
|
||||
const recordedPins = artifactPinsLockfile.packages?.[depPath].artifactPins
|
||||
expect(Object.keys(recordedPins ?? {})).toHaveLength(2)
|
||||
expect(recordedPins?.['dependency-side-effects:v1:deps=other'])
|
||||
.toEqual({ 'organization:acme': { 'other-platform': '1'.repeat(64) } })
|
||||
const [inputKey, owners] = Object.entries(recordedPins ?? {})
|
||||
.find(([key]) => key !== 'dependency-side-effects:v1:deps=other')!
|
||||
const ownerPins = owners['organization:acme']
|
||||
expect(Object.keys(ownerPins ?? {})).toHaveLength(1)
|
||||
expect(Object.keys(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(Object.values(ownerPins ?? {})[0]).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(artifactPinsChanged).toBe(1)
|
||||
expect(requestedPaths).toEqual([
|
||||
'/-/pnpr',
|
||||
'/-/pnpr/v0/artifacts/resolve',
|
||||
@@ -210,6 +252,125 @@ describe('install remote side-effects', () => {
|
||||
expect(storedFiles).toEqual([])
|
||||
expect(requestedPaths).not.toContain('/-/pnpr/v0/artifacts/blob')
|
||||
|
||||
const platformFingerprint = Object.keys(owners['organization:acme'])[0]
|
||||
const unrelatedPinLockfile: LockfileObject = {
|
||||
lockfileVersion: '9.0',
|
||||
importers: {},
|
||||
packages: {
|
||||
[depPath]: { resolution: { integrity: sourceIntegrity } },
|
||||
['stale-package@1.0.0' as DepPath]: {
|
||||
artifactPins: {
|
||||
[inputKey]: {
|
||||
'organization:acme': { [platformFingerprint]: '0'.repeat(64) },
|
||||
},
|
||||
},
|
||||
resolution: { integrity: sourceIntegrity },
|
||||
},
|
||||
},
|
||||
}
|
||||
const snapshotScoped = createRemoteSideEffectsRestorer({
|
||||
allowBuild: candidate => candidate === depPath,
|
||||
artifactPinsLockfile: unrelatedPinLockfile,
|
||||
configByUri: {},
|
||||
depsGraph,
|
||||
depsStateCache: {},
|
||||
ignoreScripts: false,
|
||||
pnprServer,
|
||||
settings: { organization: 'acme', packages: [packageName], trustedKeys },
|
||||
sideEffectsCacheRead: false,
|
||||
storeController,
|
||||
})
|
||||
const snapshotScopedFiles: PackageFilesResponse = {
|
||||
filesMap: new Map(),
|
||||
requiresBuild: true,
|
||||
resolvedFrom: 'remote',
|
||||
}
|
||||
const snapshotScopedKey = await snapshotScoped?.restore({
|
||||
graphKey,
|
||||
depPath,
|
||||
files: snapshotScopedFiles,
|
||||
name: packageName,
|
||||
resolution: { integrity: sourceIntegrity } as LockfileResolution,
|
||||
version: packageVersion,
|
||||
})
|
||||
expect(snapshotScopedKey).toBeDefined()
|
||||
|
||||
let releaseResolve!: () => void
|
||||
const waitForRelease = new Promise<void>((resolve) => {
|
||||
releaseResolve = resolve
|
||||
})
|
||||
let notifyResolveStarted!: () => void
|
||||
const resolveStarted = new Promise<void>((resolve) => {
|
||||
notifyResolveStarted = resolve
|
||||
})
|
||||
heldResolve = { wait: waitForRelease, notifyStarted: notifyResolveStarted }
|
||||
const conflictingDepPath = `${depPath}(peer@1.0.0)` as DepPath
|
||||
const envelopeDigest = owners['organization:acme'][platformFingerprint]
|
||||
const conflictingPinsLockfile: LockfileObject = {
|
||||
lockfileVersion: '9.0',
|
||||
importers: {},
|
||||
packages: {
|
||||
[depPath]: {
|
||||
artifactPins: {
|
||||
[inputKey]: {
|
||||
'organization:acme': { [platformFingerprint]: envelopeDigest },
|
||||
},
|
||||
},
|
||||
resolution: { integrity: sourceIntegrity },
|
||||
},
|
||||
[conflictingDepPath]: {
|
||||
artifactPins: {
|
||||
[inputKey]: {
|
||||
'organization:acme': { [platformFingerprint]: '0'.repeat(64) },
|
||||
},
|
||||
},
|
||||
resolution: { integrity: sourceIntegrity },
|
||||
},
|
||||
},
|
||||
}
|
||||
const collisionRestorer = createRemoteSideEffectsRestorer({
|
||||
allowBuild: candidate => candidate === depPath || candidate === conflictingDepPath,
|
||||
artifactPinsLockfile: conflictingPinsLockfile,
|
||||
configByUri: {},
|
||||
depsGraph,
|
||||
depsStateCache: {},
|
||||
ignoreScripts: false,
|
||||
pnprServer,
|
||||
settings: { organization: 'acme', packages: [packageName], trustedKeys },
|
||||
sideEffectsCacheRead: false,
|
||||
storeController,
|
||||
})!
|
||||
const firstCollisionFiles: PackageFilesResponse = {
|
||||
filesMap: new Map(),
|
||||
requiresBuild: true,
|
||||
resolvedFrom: 'remote',
|
||||
}
|
||||
const firstCollisionRestore = collisionRestorer.restore({
|
||||
graphKey,
|
||||
depPath,
|
||||
files: firstCollisionFiles,
|
||||
name: packageName,
|
||||
resolution: { integrity: sourceIntegrity } as LockfileResolution,
|
||||
version: packageVersion,
|
||||
})
|
||||
await resolveStarted
|
||||
const secondCollisionFiles: PackageFilesResponse = {
|
||||
filesMap: new Map(),
|
||||
requiresBuild: true,
|
||||
resolvedFrom: 'remote',
|
||||
}
|
||||
await expect(collisionRestorer.restore({
|
||||
graphKey,
|
||||
depPath: conflictingDepPath,
|
||||
files: secondCollisionFiles,
|
||||
name: packageName,
|
||||
resolution: { integrity: sourceIntegrity } as LockfileResolution,
|
||||
version: packageVersion,
|
||||
})).resolves.toBeUndefined()
|
||||
releaseResolve()
|
||||
await expect(firstCollisionRestore).resolves.toBeUndefined()
|
||||
expect(firstCollisionFiles.sideEffectsMaps).toBeUndefined()
|
||||
|
||||
// Restoring is per package so one package never waits on another's
|
||||
// fetch, but packages restored together still share a lookup request.
|
||||
const secondGraphKey = `${graphKey}-second`
|
||||
|
||||
@@ -314,7 +314,7 @@ describe('signed shared artifacts', () => {
|
||||
},
|
||||
})
|
||||
expect(mismatchedPackage).toEqual(new Map())
|
||||
const selected = await resolveSharedSideEffects({
|
||||
const resolveOptions = {
|
||||
registryUrl,
|
||||
authorization: 'Bearer token',
|
||||
candidates: [{
|
||||
@@ -332,13 +332,24 @@ describe('signed shared artifacts', () => {
|
||||
trustedKeys: {
|
||||
'acme-2026': publicKey.export({ format: 'der', type: 'spki' }).toString('base64'),
|
||||
},
|
||||
})
|
||||
}
|
||||
const selected = await resolveSharedSideEffects(resolveOptions)
|
||||
expect(selected.get(payload().inputKey)?.payload).toEqual(payload())
|
||||
expect(selected.get(payload().inputKey)?.envelopeDigest).toBe(
|
||||
[envelope, alternateEnvelope]
|
||||
.map(signedArtifactEnvelopeDigest)
|
||||
.sort()[0]
|
||||
)
|
||||
const pinnedDigest = signedArtifactEnvelopeDigest(alternateEnvelope)
|
||||
const pinned = await resolveSharedSideEffects({
|
||||
...resolveOptions,
|
||||
pinnedEnvelopeDigests: new Map([[payload().inputKey, pinnedDigest]]),
|
||||
})
|
||||
expect(pinned.get(payload().inputKey)?.envelopeDigest).toBe(pinnedDigest)
|
||||
await expect(resolveSharedSideEffects({
|
||||
...resolveOptions,
|
||||
pinnedEnvelopeDigests: new Map([[payload().inputKey, '0'.repeat(64)]]),
|
||||
})).resolves.toEqual(new Map())
|
||||
await expect(downloadSharedArtifactBlob({
|
||||
registryUrl,
|
||||
authorization: 'Bearer token',
|
||||
@@ -348,6 +359,8 @@ describe('signed shared artifacts', () => {
|
||||
'/-/pnpr/v0/artifacts',
|
||||
'/-/pnpr/v0/artifacts/resolve',
|
||||
'/-/pnpr/v0/artifacts/resolve',
|
||||
'/-/pnpr/v0/artifacts/resolve',
|
||||
'/-/pnpr/v0/artifacts/resolve',
|
||||
'/-/pnpr/v0/artifacts/blob',
|
||||
])
|
||||
expect(requests.every(request => request.authorization === 'Bearer token')).toBe(true)
|
||||
|
||||
Reference in new issue
Block a user