fix(lockfile): compute tarball integrity upon download (#12491)

Some registries generate tarballs on demand and cannot list an integrity in
their packument. pnpm then wrote integrity-less lockfile entries on the first
install and failed the next one with ERR_PNPM_MISSING_TARBALL_INTEGRITY, unable
to install from its own lockfile.

Compute the missing integrity from the downloaded bytes and write it into the
resolution before the lockfile is built:

- Add an optional `resolutionNeedsFetch` contract to the fetcher API (backward
  compatible, since custom fetchers come from hooks). The remote-tarball fetcher
  reports it when a resolution lacks integrity; the picked fetcher's signal flows
  through PackageResponse -> ResolvedPackage so nothing re-derives it.
- The package requester downloads such tarballs (including under --lockfile-only /
  skipFetch / not-installable) and fills the computed integrity onto the resolution
  via the already-running `fetching` promise, so dependency resolution isn't
  blocked. The deps-resolver awaits only the flagged entries before updateLockfile,
  because the integrity feeds the global virtual-store paths.
- Move read-side enforcement into the npm resolver's lockfile verifier
  (MISSING_TARBALL_INTEGRITY): reject a registry/http(s) tarball entry whose
  integrity is missing/empty/non-string, fail-closed, before the URL-keyed and
  semver short-circuits. Drop the earlier read-side auto-heal (a missing-field
  bypass). Harden against tampered lockfiles (non-string tarball/integrity).
- Reuse the fetcher picked during resolution on the fetch path instead of running
  pickFetcher (and a custom fetcher's async canFetch) twice per package.

Mirrored in pacquet: PrefetchingResolver computes the integrity for integrity-less
tarball resolutions during resolution (FetchTarballForResolution::run), deduped per
URL with a singleflight cache.

Closes pnpm/pnpm#12145.

---------

Co-authored-by: Zoltan Kochan <z@kochan.io>
This commit is contained in:
Ajeet D'Souza
2026-06-22 15:32:29 +05:30
committed by GitHub
parent d1635db8b3
commit bae694f639
43 changed files with 1243 additions and 238 deletions

View File

@@ -0,0 +1,19 @@
---
"@pnpm/resolving.npm-resolver": minor
"@pnpm/resolving.resolver-base": minor
"@pnpm/fetching.fetcher-base": minor
"@pnpm/hooks.types": minor
"@pnpm/installing.package-requester": minor
"@pnpm/installing.context": patch
"@pnpm/installing.deps-resolver": patch
"@pnpm/fetching.tarball-fetcher": patch
"@pnpm/fetching.pick-fetcher": patch
"@pnpm/store.controller-types": patch
"@pnpm/lockfile.utils": minor
"@pnpm/deps.graph-builder": patch
"@pnpm/installing.deps-restorer": patch
"@pnpm/patching.commands": patch
"pnpm": minor
---
Some registries generate tarballs on-demand and cannot provide an integrity checksum in their package metadata. In that case pnpm now computes the integrity from the downloaded tarball and stores it in the lockfile, so the entry is verifiable on subsequent installs instead of being written without an integrity (which would fail the next install). This also applies to `--lockfile-only`: the tarball is downloaded so its integrity can be computed. A lockfile entry that is still missing its integrity is rejected as a `ERR_PNPM_MISSING_TARBALL_INTEGRITY` lockfile verification violation (the install fails closed) rather than being silently re-fetched.

View File

@@ -683,7 +683,7 @@ importers:
packages:
acme@1.0.0:
resolution: {integrity: sha512-deadbeef, tarball: 'https://codeload.github.com/org/acme/tar.gz/abc123', gitHosted: false}
resolution: {integrity: sha512-deadbeef, tarball: 'https://codeload.github.com/org/acme/tar.gz/0123456789abcdef0123456789abcdef01234567', gitHosted: false}
snapshots:
@@ -756,7 +756,7 @@ importers:
packages:
acme@1.0.0:
resolution: {integrity: sha512-deadbeef, tarball: 'https://CODELOAD.GITHUB.COM/org/acme/tar.gz/abc123', gitHosted: false}
resolution: {integrity: sha512-deadbeef, tarball: 'https://CODELOAD.GITHUB.COM/org/acme/tar.gz/0123456789abcdef0123456789abcdef01234567', gitHosted: false}
snapshots:

View File

@@ -452,21 +452,89 @@ impl From<ResolutionSerde> for LockfileResolution {
}
}
/// Best-effort URL-prefix check used to back-fill `gitHosted` on tarball
/// resolutions written by older pnpm versions, and to gate trust on the
/// tarball URL rather than the (tamper-prone) `gitHosted` flag. Mirrors
/// upstream's `isGitHostedTarballUrl` at
/// <https://github.com/pnpm/pnpm/blob/94240bc046/lockfile/fs/src/lockfileFormatConverters.ts#L23-L29>.
/// Recognizes immutable archive URLs emitted by known git providers. The result
/// gates integrity exemptions, so path shapes are matched explicitly and refs
/// must be full commit SHAs.
#[must_use]
pub fn is_git_hosted_tarball_url(url: &str) -> bool {
// Schemes and hostnames are case-insensitive, so match against a lowercased
// copy: a tampered `https://CODELOAD.GITHUB.COM/...` must not slip past as a
// non-git-hosted (and therefore registry-trusted) tarball.
let lower = url.to_ascii_lowercase();
(lower.starts_with("https://codeload.github.com/")
|| lower.starts_with("https://bitbucket.org/")
|| lower.starts_with("https://gitlab.com/"))
&& lower.contains("tar.gz")
let Some((host, path, query)) = parse_https_url(url) else { return false };
if host.eq_ignore_ascii_case("codeload.github.com") {
return is_github_codeload_archive(path);
}
if host.eq_ignore_ascii_case("bitbucket.org") {
return is_bitbucket_archive(path);
}
if host.eq_ignore_ascii_case("gitlab.com") {
return is_gitlab_archive(path, query);
}
false
}
fn parse_https_url(url: &str) -> Option<(&str, &str, Option<&str>)> {
const HTTPS_SCHEME: &str = "https://";
if !url.get(..HTTPS_SCHEME.len())?.eq_ignore_ascii_case(HTTPS_SCHEME) {
return None;
}
let rest = url.get(HTTPS_SCHEME.len()..)?;
let (host, path_and_query) = rest.split_once('/')?;
let path_and_query = path_and_query.split_once('#').map_or(path_and_query, |(path, _)| path);
let (path, query) = path_and_query
.split_once('?')
.map_or((path_and_query, None), |(path, query)| (path, Some(query)));
Some((host, path, query))
}
fn is_github_codeload_archive(path: &str) -> bool {
let segments = path_segments(path);
segments.len() == 4 && segments[2] == "tar.gz" && is_full_commit_sha(segments[3])
}
fn is_bitbucket_archive(path: &str) -> bool {
let segments = path_segments(path);
if segments.len() != 4 || segments[2] != "get" {
return false;
}
let Some(commit) = segments[3].strip_suffix(".tar.gz") else { return false };
is_full_commit_sha(commit)
}
fn is_gitlab_archive(path: &str, query: Option<&str>) -> bool {
let segments = path_segments(path);
if segments.len() == 6
&& segments[0] == "api"
&& segments[1] == "v4"
&& segments[2] == "projects"
&& segments[4] == "repository"
&& segments[5] == "archive.tar.gz"
{
return query_param(query, "ref").is_some_and(is_full_commit_sha);
}
let Some(archive_marker_index) =
segments.windows(2).position(|window| window[0] == "-" && window[1] == "archive")
else {
return false;
};
if archive_marker_index < 2 || segments.len() != archive_marker_index + 4 {
return false;
}
let commit = segments[archive_marker_index + 2];
let archive_name = segments[archive_marker_index + 3];
archive_name.ends_with(".tar.gz") && is_full_commit_sha(commit)
}
fn path_segments(path: &str) -> Vec<&str> {
path.split('/').filter(|segment| !segment.is_empty()).collect()
}
fn query_param<'query>(query: Option<&'query str>, key: &str) -> Option<&'query str> {
query?.split('&').find_map(|part| {
let (part_key, value) = part.split_once('=')?;
(part_key == key).then_some(value)
})
}
fn is_full_commit_sha(value: &str) -> bool {
value.len() == 40 && value.as_bytes().iter().all(u8::is_ascii_hexdigit)
}
impl From<LockfileResolution> for ResolutionSerde {

View File

@@ -1,8 +1,8 @@
use super::{
BinaryArchive, BinaryResolution, BinarySpec, DirectoryResolution, GitResolution,
LockfileResolution, PlatformAssetResolution, PlatformAssetTarget, PlatformSelector,
RegistryResolution, TarballResolution, VariationsResolution, libc_matches,
select_platform_variant,
RegistryResolution, TarballResolution, VariationsResolution, is_git_hosted_tarball_url,
libc_matches, select_platform_variant,
};
use crate::serialize_yaml;
use pretty_assertions::assert_eq;
@@ -10,6 +10,8 @@ use ssri::Integrity;
use std::collections::BTreeMap;
use text_block_macros::text_block;
const GIT_COMMIT: &str = "0123456789abcdef0123456789abcdef01234567";
fn integrity(integrity_str: &str) -> Integrity {
integrity_str.parse().expect("parse integrity string")
}
@@ -87,13 +89,11 @@ fn deserialize_tarball_resolution_with_git_hosted() {
#[test]
fn deserialize_tarball_resolution_backfills_git_hosted() {
eprintln!("CASE: codeload.github.com");
let yaml = text_block! {
"tarball: https://codeload.github.com/foo/bar/tar.gz/abc1234"
};
let received: LockfileResolution = serde_saphyr::from_str(yaml).unwrap();
let yaml = format!("tarball: https://codeload.github.com/foo/bar/tar.gz/{GIT_COMMIT}");
let received: LockfileResolution = serde_saphyr::from_str(&yaml).unwrap();
dbg!(&received);
let expected = LockfileResolution::Tarball(TarballResolution {
tarball: "https://codeload.github.com/foo/bar/tar.gz/abc1234".to_string(),
tarball: format!("https://codeload.github.com/foo/bar/tar.gz/{GIT_COMMIT}"),
integrity: None,
git_hosted: Some(true),
path: None,
@@ -101,12 +101,14 @@ fn deserialize_tarball_resolution_backfills_git_hosted() {
assert_eq!(received, expected);
eprintln!("CASE: gitlab.com archive");
let yaml = text_block! {
"tarball: https://gitlab.com/foo/bar/-/archive/abc1234/bar-abc1234.tar.gz"
};
let received: LockfileResolution = serde_saphyr::from_str(yaml).unwrap();
let yaml = format!(
"tarball: https://gitlab.com/foo/bar/-/archive/{GIT_COMMIT}/bar-{GIT_COMMIT}.tar.gz",
);
let received: LockfileResolution = serde_saphyr::from_str(&yaml).unwrap();
let expected = LockfileResolution::Tarball(TarballResolution {
tarball: "https://gitlab.com/foo/bar/-/archive/abc1234/bar-abc1234.tar.gz".to_string(),
tarball: format!(
"https://gitlab.com/foo/bar/-/archive/{GIT_COMMIT}/bar-{GIT_COMMIT}.tar.gz",
),
integrity: None,
git_hosted: Some(true),
path: None,
@@ -114,12 +116,10 @@ fn deserialize_tarball_resolution_backfills_git_hosted() {
assert_eq!(received, expected);
eprintln!("CASE: bitbucket.org archive");
let yaml = text_block! {
"tarball: https://bitbucket.org/foo/bar/get/abc1234.tar.gz"
};
let received: LockfileResolution = serde_saphyr::from_str(yaml).unwrap();
let yaml = format!("tarball: https://bitbucket.org/foo/bar/get/{GIT_COMMIT}.tar.gz");
let received: LockfileResolution = serde_saphyr::from_str(&yaml).unwrap();
let expected = LockfileResolution::Tarball(TarballResolution {
tarball: "https://bitbucket.org/foo/bar/get/abc1234.tar.gz".to_string(),
tarball: format!("https://bitbucket.org/foo/bar/get/{GIT_COMMIT}.tar.gz"),
integrity: None,
git_hosted: Some(true),
path: None,
@@ -153,6 +153,25 @@ fn deserialize_tarball_resolution_backfills_git_hosted() {
assert_eq!(received, expected);
}
#[test]
fn is_git_hosted_tarball_url_rejects_false_positives() {
assert!(is_git_hosted_tarball_url(&format!(
"https://codeload.github.com/foo/bar/tar.gz/{GIT_COMMIT}"
)));
assert!(is_git_hosted_tarball_url(&format!(
"https://gitlab.com/api/v4/projects/foo%2Fbar/repository/archive.tar.gz?ref={GIT_COMMIT}"
)));
assert!(!is_git_hosted_tarball_url("https://gitlab.com/foo/bar?download=tar.gz"));
assert!(!is_git_hosted_tarball_url("https://codeload.github.com/foo/bar/tar.gz/main"));
assert!(!is_git_hosted_tarball_url(
"https://gitlab.com/foo/bar/-/archive/main/bar-main.tar.gz",
));
assert!(!is_git_hosted_tarball_url(
"https://gitlab.com/api/v4/projects/foo%2Fbar/repository/archive.tar.gz",
));
assert!(!is_git_hosted_tarball_url("https://bitbucket.org/foo/bar/get/main.tar.gz"));
}
#[test]
fn serialize_tarball_resolution() {
eprintln!("CASE: without integrity");

View File

@@ -27,19 +27,25 @@ use crate::{
install_package_from_registry::{extract_tarball, manifest_file_count, manifest_unpacked_size},
retry_config::retry_opts_from_config,
};
use dashmap::DashSet;
use dashmap::{DashMap, DashSet};
use pacquet_config::Config;
use pacquet_lockfile::{LockfileResolution, is_git_hosted_tarball_url};
use pacquet_network::{AuthHeaders, ThrottledClient};
use pacquet_reporter::Reporter;
use pacquet_reporter::{Reporter, SilentReporter};
use pacquet_resolving_resolver_base::{
LatestQuery, ResolveFuture, ResolveLatestFuture, ResolveOptions, ResolveResult, Resolver,
WantedDependency,
LatestQuery, ResolveError, ResolveFuture, ResolveLatestFuture, ResolveOptions, ResolveResult,
Resolver, WantedDependency,
};
use pacquet_store_dir::{
SharedReadonlyStoreIndex, SharedVerifiedFilesCache, StoreDir, StoreIndexWriter,
};
use pacquet_tarball::{DownloadTarballToStore, MemCache, RetryOpts, SharedReportedProgressKeys};
use pacquet_tarball::{
DownloadTarballToStore, FetchTarballForResolution, MemCache, RetryOpts,
SharedReportedProgressKeys,
};
use ssri::Integrity;
use std::{marker::PhantomData, sync::Arc};
use tokio::sync::OnceCell;
/// Borrowed-data bag handed to [`PrefetchingResolver::new`]. Everything
/// the wrapper needs to drive a background tarball download:
@@ -92,6 +98,10 @@ struct OwnedFetchCtx {
/// without this gate the bench saw ~3-5k redundant spawns per
/// install on the alotta-files fixture (one per dependent edge).
spawned_urls: Arc<DashSet<String>>,
/// Per-URL singleflight cache for integrity-less tarballs. The first
/// edge downloads and computes the integrity; later edges await the
/// same cell instead of fetching the URL again.
integrity_cache: Arc<DashMap<String, Arc<OnceCell<Integrity>>>>,
}
/// Wraps an inner [`Resolver`] and, after each successful resolve that
@@ -146,10 +156,71 @@ impl<Reporter: self::Reporter + 'static> PrefetchingResolver<Reporter> {
verify_store_integrity: config.verify_store_integrity,
progress_reported: SharedReportedProgressKeys::clone(progress_reported),
spawned_urls: Arc::new(DashSet::new()),
integrity_cache: Arc::new(DashMap::new()),
};
PrefetchingResolver { inner, ctx, _phantom: PhantomData }
}
/// Populate remote tarball resolutions whose integrity can only be
/// learned from the downloaded bytes. `file:` and git-hosted tarballs
/// are anchored by local bytes or a commit SHA and remain unchanged.
async fn populate_missing_integrity(
&self,
result: &mut ResolveResult,
) -> Result<(), ResolveError> {
let LockfileResolution::Tarball(tarball) = &result.resolution else {
return Ok(());
};
if tarball.integrity.is_some()
// git-hosted tarballs are anchored by their commit SHA, not an integrity. Detect
// them by URL, NOT by the `git_hosted` flag: the flag is tamper-prone lockfile
// input, so trusting it would let a forged `git_hosted: true` on an arbitrary URL
// skip the integrity computation. A real git-hosted archive (codeload/gitlab/
// bitbucket) always has a matching URL.
|| is_git_hosted_tarball_url(&tarball.tarball)
|| tarball.tarball.starts_with("file:")
{
return Ok(());
}
let package_url = tarball.tarball.clone();
// Scope credentials are selected from `name@version` when the
// resolver knows it; direct URL tarballs fall back to URL identity.
let package_id = result
.name_ver
.as_ref()
.map_or_else(|| package_url.clone(), |nv| format!("{}@{}", nv.name, nv.suffix));
// Singleflight per URL: the same integrity-less tarball can arrive on many edges,
// so compute its integrity once and share it. Clone the cell's `Arc` out of the map
// before awaiting so the shard lock isn't held across the download.
let cell = Arc::clone(&self.ctx.integrity_cache.entry(package_url.clone()).or_default());
let integrity = cell
.get_or_try_init(|| async {
// This fetch warms the mem cache, so the prefetch path should not
// spawn another task for the same URL.
self.ctx.spawned_urls.insert(package_url.clone());
let resolved = FetchTarballForResolution {
http_client: &self.ctx.http_client,
store_dir: self.ctx.store_dir,
store_index_writer: self.ctx.store_index_writer.clone(),
package_url: &package_url,
package_id: &package_id,
auth_headers: &self.ctx.auth_headers,
retry_opts: self.ctx.retry_opts,
}
.run::<SilentReporter>(Some(&self.ctx.mem_cache))
.await
.map_err(|err| Box::new(err) as ResolveError)?;
Ok::<_, ResolveError>(resolved.integrity)
})
.await?
.clone();
if let LockfileResolution::Tarball(tarball) = &mut result.resolution {
tarball.integrity = Some(integrity);
}
Ok(())
}
/// Inspect a fresh `ResolveResult` and, if it carries a tarball
/// URL + integrity, kick off the download as a detached
/// [`tokio::spawn`] task.
@@ -263,9 +334,10 @@ impl<Reporter: self::Reporter + 'static> Resolver for PrefetchingResolver<Report
opts: &'a ResolveOptions,
) -> ResolveFuture<'a> {
Box::pin(async move {
let result = self.inner.resolve(wanted_dependency, opts).await?;
if let Some(result_ref) = result.as_ref() {
self.maybe_kickoff_download(result_ref);
let mut result = self.inner.resolve(wanted_dependency, opts).await?;
if let Some(result_mut) = result.as_mut() {
self.populate_missing_integrity(result_mut).await?;
self.maybe_kickoff_download(result_mut);
}
Ok(result)
})

View File

@@ -171,6 +171,9 @@ impl TarballResolver {
store_dir: ctx.store_dir,
store_index_writer: ctx.store_index_writer.clone(),
package_url: &resolved_url,
// A direct https tarball has no resolver-known name@version, so the URL is the
// only identifier; such tarballs carry no scoped-registry auth.
package_id: &resolved_url,
auth_headers: &ctx.auth_headers,
retry_opts: ctx.retry_opts,
}

View File

@@ -2291,6 +2291,9 @@ pub struct FetchTarballForResolution<'a> {
pub store_dir: &'static StoreDir,
pub store_index_writer: Option<Arc<StoreIndexWriter>>,
pub package_url: &'a str,
/// Package identity used for scoped auth lookup and the intermediate
/// store-index cache key.
pub package_id: &'a str,
pub auth_headers: &'a AuthHeaders,
pub retry_opts: RetryOpts,
}
@@ -2305,25 +2308,21 @@ impl FetchTarballForResolution<'_> {
store_dir,
store_index_writer,
package_url,
package_id,
auth_headers,
retry_opts,
} = self;
// `None` expected-integrity → compute it from the bytes. The
// package_id / requester are the post-redirect URL: the real
// `name@version` is only known once the manifest is read below,
// and the resolve-time fetch is silent (the install pass owns
// the reporter ordering), so the placeholder never surfaces.
// `UNPRIORITIZED`: this fetch gates the resolver's walk (a
// tarball dep's manifest comes from its archive), so like a
// packument fetch it must not queue behind sized downloads.
// Resolve-time tarball fetches compute integrity from bytes and
// gate the dependency walk, so they use the same priority class as
// packument requests instead of queuing behind sized downloads.
let (integrity, cas_paths, pkg_files_idx) = fetch_and_extract_with_retry::<Reporter>(
http_client,
package_url,
None,
None,
UNPRIORITIZED,
package_url,
package_id,
package_url,
store_dir,
retry_opts,

View File

@@ -1,8 +1,9 @@
use super::{
DownloadTarballToStore, HttpStatusError, MemCache, NetworkError, PrefetchedCasPaths, RetryOpts,
SharedReportedProgressKeys, TarballError, VerifyChecksumError, allocate_tarball_buffer,
download_priority, extract_tarball_entries, extract_zip_entries, fetch_and_extract_with_retry,
is_transient_error, normalize_bundled_manifest, prefetch_cas_paths,
DownloadTarballToStore, FetchTarballForResolution, HttpStatusError, MemCache, NetworkError,
PrefetchedCasPaths, RetryOpts, SharedReportedProgressKeys, TarballError, VerifyChecksumError,
allocate_tarball_buffer, download_priority, extract_tarball_entries, extract_zip_entries,
fetch_and_extract_with_retry, is_transient_error, normalize_bundled_manifest,
prefetch_cas_paths,
};
use pacquet_network::{AuthHeaders, ThrottledClient, UNPRIORITIZED};
use pacquet_reporter::SilentReporter;
@@ -1250,6 +1251,81 @@ async fn retries_integrity_mismatch_until_exhausted() {
drop(store_dir_keep);
}
/// Integrity-less tarball resolutions must be completed from the
/// downloaded bytes before they are written to the lockfile.
#[tokio::test]
async fn fetch_for_resolution_computes_integrity_when_none_is_expected() {
let (store_dir_keep, store_path) = tempdir_with_leaked_path();
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/pkg.tgz")
.with_status(200)
.with_body(FASTIFY_ERROR_TARBALL)
.expect(1)
.create_async()
.await;
let url = format!("{}/pkg.tgz", server.url());
let client = ThrottledClient::default();
let resolved = FetchTarballForResolution {
http_client: &client,
store_dir: store_path,
store_index_writer: None,
package_url: &url,
package_id: &url,
auth_headers: &AuthHeaders::default(),
retry_opts: fast_retry_opts(),
}
.run::<SilentReporter>(None)
.await
.expect("a registry that omits integrity should get it computed from the bytes");
assert_eq!(resolved.integrity, integrity(FASTIFY_ERROR_INTEGRITY));
mock.assert_async().await;
drop(store_dir_keep);
}
/// `FetchTarballForResolution` must forward its `package_id` (the package's
/// `name@version`) for auth/scope selection, so a private scoped registry tarball
/// resolves its scope token while its integrity is computed during resolution.
#[tokio::test]
async fn fetch_for_resolution_uses_package_id_for_scoped_auth() {
let (store_dir_keep, store_path) = tempdir_with_leaked_path();
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/pkg.tgz")
.match_header("authorization", "Bearer scoped-token")
.with_status(200)
.with_body(FASTIFY_ERROR_TARBALL)
.expect(1)
.create_async()
.await;
let url = format!("{}/pkg.tgz", server.url());
let client = ThrottledClient::default();
let registry_key = format!("{}@scope", pacquet_network::nerf_dart(&server.url()));
let auth_headers =
AuthHeaders::from_creds_map([(registry_key, "Bearer scoped-token".to_owned())], None);
let resolved = FetchTarballForResolution {
http_client: &client,
store_dir: store_path,
store_index_writer: None,
package_url: &url,
package_id: "@scope/test-pkg@1.0.0",
auth_headers: &auth_headers,
retry_opts: fast_retry_opts(),
}
.run::<SilentReporter>(None)
.await
.expect("the scope token selected via package_id should let the fetch succeed");
assert_eq!(resolved.integrity, integrity(FASTIFY_ERROR_INTEGRITY));
mock.assert_async().await;
drop(store_dir_keep);
}
/// 404 is in pnpm's no-retry set. `expect(1)` makes the test fail if
/// the retry loop fires a second request — that would mean we're
/// spinning on a permanently-missing tarball.

9
pnpm-lock.yaml generated
View File

@@ -5547,6 +5547,9 @@ importers:
'@pnpm/core-loggers':
specifier: workspace:*
version: link:../../core/core-loggers
'@pnpm/error':
specifier: workspace:*
version: link:../../core/error
'@pnpm/installing.modules-yaml':
specifier: workspace:*
version: link:../modules-yaml
@@ -7458,9 +7461,6 @@ importers:
'@pnpm/error':
specifier: workspace:*
version: link:../../core/error
'@pnpm/fetching.pick-fetcher':
specifier: workspace:*
version: link:../../fetching/pick-fetcher
'@pnpm/fs.packlist':
specifier: workspace:*
version: link:../../fs/packlist
@@ -8890,6 +8890,9 @@ importers:
specifier: workspace:*
version: link:../../core/types
devDependencies:
'@jest/globals':
specifier: 'catalog:'
version: 30.4.1
'@pnpm/resolving.resolver-base':
specifier: workspace:*
version: 'link:'

View File

@@ -28,11 +28,19 @@ export interface FetchOptions {
ignoreFilePattern?: string
}
export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = (
export interface ResolutionFetchContract {
/**
* Returns `true` when a resolution is missing data only this fetcher can supply, so
* callers must fetch before treating the resolution as complete.
*/
resolutionNeedsFetch?: (resolution: Resolution) => boolean
}
export type FetchFunction<FetcherResolution = Resolution, Options = FetchOptions, Result = FetchResult> = ((
cafs: Cafs,
resolution: FetcherResolution,
opts: Options
) => Promise<Result>
) => Promise<Result>) & ResolutionFetchContract
export interface FetchResult {
local?: boolean

View File

@@ -1,9 +1,19 @@
import { PnpmError } from '@pnpm/error'
import type { BinaryFetcher, DirectoryFetcher, Fetchers, FetchFunction, FetchOptions, GitFetcher } from '@pnpm/fetching.fetcher-base'
import type {
BinaryFetcher,
DirectoryFetcher,
Fetchers,
FetchFunction,
FetchOptions,
FetchResult,
GitFetcher,
} from '@pnpm/fetching.fetcher-base'
import type { CustomFetcher } from '@pnpm/hooks.types'
import type { AtomicResolution } from '@pnpm/resolving.resolver-base'
import { type AtomicResolution, classifyResolution } from '@pnpm/resolving.resolver-base'
import type { Cafs } from '@pnpm/store.cafs-types'
export type PickedFetcher = FetchFunction | DirectoryFetcher | GitFetcher | BinaryFetcher
export async function pickFetcher (
fetcherByHostingType: Fetchers,
resolution: AtomicResolution,
@@ -11,7 +21,7 @@ export async function pickFetcher (
customFetchers?: CustomFetcher[]
packageId: string
}
): Promise<FetchFunction | DirectoryFetcher | GitFetcher | BinaryFetcher> {
): Promise<PickedFetcher> {
// Try custom fetcher hooks first if available
// Custom fetchers act as complete fetcher replacements
if (opts?.customFetchers && opts.customFetchers.length > 0) {
@@ -21,41 +31,23 @@ export async function pickFetcher (
const canFetch = await customFetcher.canFetch(opts.packageId, resolution)
if (canFetch) {
// Return a wrapper FetchFunction that calls the custom fetcher's fetch method
// The custom fetcher's fetch receives cafs, resolution, opts, and the standard fetchers for delegation
return async (cafs: Cafs, resolution: AtomicResolution, fetchOpts: FetchOptions) => {
return customFetcher.fetch!(cafs, resolution, fetchOpts, fetcherByHostingType)
}
// Preserve `this` for custom fetchers that implement their optional
// resolution contract as a method.
const resolutionNeedsFetch = typeof customFetcher.resolutionNeedsFetch === 'function'
? customFetcher.resolutionNeedsFetch.bind(customFetcher)
: undefined
return Object.assign(
async (cafs: Cafs, resolution: AtomicResolution, fetchOpts: FetchOptions): Promise<FetchResult> =>
customFetcher.fetch!(cafs, resolution, fetchOpts, fetcherByHostingType),
{ resolutionNeedsFetch }
) as FetchFunction
}
}
}
}
// No custom fetcher handled the fetch, use standard fetcher selection
let fetcherType: keyof Fetchers | undefined
// Determine the fetcher type based on resolution
if (resolution.type == null) {
// Tarball resolution without explicit type
if ('tarball' in resolution && resolution.tarball) {
if (resolution.tarball.startsWith('file:')) {
fetcherType = 'localTarball'
} else if (
('gitHosted' in resolution && resolution.gitHosted === true) ||
// URL fallback for resolutions that didn't go through the resolver or
// the lockfile loader (e.g., constructed ad-hoc).
isGitHostedPkgUrl(resolution.tarball)
) {
fetcherType = 'gitHostedTarball'
} else {
fetcherType = 'remoteTarball'
}
}
} else if (resolution.type === 'directory' || resolution.type === 'git' || resolution.type === 'binary') {
// Standard resolution types that map directly to fetchers
fetcherType = resolution.type
} else {
// Custom resolution type that wasn't handled by any custom fetcher
const fetcherType = classifyResolution(resolution)
if (fetcherType === 'custom') {
throw new PnpmError(
'UNSUPPORTED_RESOLUTION_TYPE',
`Cannot fetch dependency with custom resolution type "${resolution.type}". ` +
@@ -63,7 +55,7 @@ export async function pickFetcher (
)
}
const fetch = fetcherType != null ? fetcherByHostingType[fetcherType] : undefined
const fetch = fetcherByHostingType[fetcherType]
if (!fetch) {
throw new Error(`Fetching for dependency type "${resolution.type ?? 'tarball'}" is not supported`)
@@ -71,11 +63,3 @@ export async function pickFetcher (
return fetch
}
export function isGitHostedPkgUrl (url: string): boolean {
return (
url.startsWith('https://codeload.github.com/') ||
url.startsWith('https://bitbucket.org/') ||
url.startsWith('https://gitlab.com/')
) && url.includes('tar.gz')
}

View File

@@ -590,3 +590,32 @@ describe('custom fetcher implementation examples', () => {
})
})
})
test('remoteTarball fetcher reports resolutionNeedsFetch from the integrity', () => {
const fetchers = createTarballFetcher(createFetchFromRegistry({}), () => undefined, { storeIndex })
expect(fetchers.remoteTarball.resolutionNeedsFetch?.(createMockResolution({ tarball: 'http://x/p.tgz' }))).toBe(true)
expect(fetchers.remoteTarball.resolutionNeedsFetch?.(createMockResolution({ tarball: 'http://x/p.tgz', integrity: 'sha512-x' }))).toBe(false)
// Empty/non-string integrity from a tampered lockfile counts as missing.
expect(fetchers.remoteTarball.resolutionNeedsFetch?.(createMockResolution({ tarball: 'http://x/p.tgz', integrity: '' }))).toBe(true)
expect(fetchers.remoteTarball.resolutionNeedsFetch?.(createMockResolution({ tarball: 'http://x/p.tgz', integrity: true }))).toBe(true)
// file: and git-hosted tarballs are anchored otherwise and don't force a fetch.
expect(fetchers.localTarball.resolutionNeedsFetch).toBeUndefined()
expect(fetchers.gitHostedTarball.resolutionNeedsFetch).toBeUndefined()
})
test('pickFetcher() forwards a custom fetcher resolutionNeedsFetch hook bound to the fetcher', async () => {
const customFetcher = {
marker: 'mine',
canFetch: () => true,
fetch: jest.fn() as unknown as CustomFetcher['fetch'],
resolutionNeedsFetch (this: { marker: string }): boolean {
return this.marker === 'mine'
},
}
const picked = await pickFetcher(
createMockFetchers(),
createMockResolution({ tarball: 'http://example.com/p.tgz' }),
{ customFetchers: [customFetcher as unknown as CustomFetcher], packageId: 'p@1.0.0' }
) as FetchFunction
expect(picked.resolutionNeedsFetch?.(createMockResolution({}))).toBe(true)
})

View File

@@ -32,7 +32,7 @@ test('should pick remoteTarball fetcher', async () => {
test.each([
'https://codeload.github.com/zkochan/is-negative/tar.gz/6dcce91c268805d456b8a575b67d7febc7ae2933',
'https://bitbucket.org/pnpmjs/git-resolver/get/87cf6a67064d2ce56e8cd20624769a5512b83ff9.tar.gz',
'https://gitlab.com/api/v4/projects/pnpm%2Fgit-resolver/repository/archive.tar.gz',
'https://gitlab.com/api/v4/projects/pnpm%2Fgit-resolver/repository/archive.tar.gz?ref=988c61e11dc8d9ca0b5580cb15291951812549dc',
'https://gitlab.com/pnpm/git-resolver/-/archive/988c61e11dc8d9ca0b5580cb15291951812549dc/git-resolver-988c61e11dc8d9ca0b5580cb15291951812549dc.tar.gz',
])('should pick gitHostedTarball fetcher', async (tarball) => {
const gitHostedTarball = jest.fn() as FetchFunction

View File

@@ -60,6 +60,10 @@ export function createTarballFetcher (
offline: opts.offline,
storeIndex: opts.storeIndex,
}) as FetchFunction
// Missing integrity is the only remote-tarball case that must fetch before store reuse.
remoteTarballFetcher.resolutionNeedsFetch = (resolution) => {
return getExpectedIntegrity(resolution) == null
}
return {
localTarball: createLocalTarballFetcher(opts.storeIndex),
@@ -91,7 +95,7 @@ async function fetchFromTarball (
getAuthHeaderByURI: ctx.getAuthHeaderByURI,
cafs,
storeIndex: ctx.storeIndex,
integrity: resolution.integrity,
integrity: getExpectedIntegrity(resolution),
readManifest: opts.readManifest,
onProgress: opts.onProgress,
onStart: opts.onStart,
@@ -102,3 +106,10 @@ async function fetchFromTarball (
ignoreFilePattern: opts.ignoreFilePattern,
})
}
function getExpectedIntegrity (resolution: unknown): string | undefined {
const integrity = (resolution as { integrity?: unknown }).integrity
return typeof integrity === 'string' && integrity.length > 0
? integrity
: undefined
}

View File

@@ -310,6 +310,30 @@ test('retry when integrity check fails', async () => {
expect(params[1]).toStrictEqual([tarballSize, 2])
})
test('computes integrity when the expected integrity is not a string', async () => {
const tarballContent = fs.readFileSync(tarballPath)
const mockPool = mockAgent.get(registry)
mockPool.intercept({ path: '/foo.tgz', method: 'GET' }).reply(200, tarballContent, {
headers: { 'Content-Length': tarballSize.toString() },
})
process.chdir(temporaryDirectory())
const resolution = {
integrity: true,
tarball: `${registry}/foo.tgz`,
} as unknown as Parameters<typeof fetch.remoteTarball>[1]
const result = await fetch.remoteTarball(cafs, resolution, {
filesIndexFile,
lockfileDir: process.cwd(),
pkg,
})
expect(result.integrity).toMatch(/^sha512-/)
})
test('fail when integrity check of local file fails', async () => {
const storeDir = temporaryDirectory()
process.chdir(storeDir)

View File

@@ -1,4 +1,4 @@
import type { Fetchers, FetchOptions, FetchResult } from '@pnpm/fetching.fetcher-base'
import type { Fetchers, FetchOptions, FetchResult, ResolutionFetchContract } from '@pnpm/fetching.fetcher-base'
import type { LockfileObject, PackageSnapshot } from '@pnpm/lockfile.types'
import type { Resolution, WantedDependency } from '@pnpm/resolving.resolver-base'
import type { Cafs } from '@pnpm/store.cafs-types'
@@ -86,7 +86,7 @@ export interface CustomResolver {
shouldRefreshResolution?: (depPath: string, pkgSnapshot: PackageSnapshot) => boolean | Promise<boolean>
}
export interface CustomFetcher {
export interface CustomFetcher extends ResolutionFetchContract {
/**
* Called to determine if this fetcher should handle fetching a package.
* This is called for each package that needs to be fetched.

View File

@@ -34,6 +34,7 @@
"dependencies": {
"@pnpm/constants": "workspace:*",
"@pnpm/core-loggers": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/installing.modules-yaml": "workspace:*",
"@pnpm/installing.read-projects-context": "workspace:*",
"@pnpm/lockfile.fs": "workspace:*",

View File

@@ -1,7 +1,10 @@
import path from 'node:path'
import {
LOCKFILE_VERSION,
WANTED_LOCKFILE,
} from '@pnpm/constants'
import { PnpmError } from '@pnpm/error'
import {
createLockfileObject,
existsNonEmptyWantedLockfile,
@@ -64,7 +67,9 @@ export async function readLockfiles (
}
const fileReads = [] as Array<Promise<LockfileObject | undefined | null>>
let lockfileHadConflicts: boolean = false
let wantedLockfileFileExists = false
if (opts.useLockfile) {
wantedLockfileFileExists = await existsNonEmptyWantedLockfile(opts.lockfileDir, lockfileOpts)
if (!opts.frozenLockfile) {
fileReads.push(
(async () => {
@@ -107,6 +112,9 @@ export async function readLockfiles (
})()
)
const files = await Promise.all<LockfileObject | null | undefined>(fileReads)
if (opts.frozenLockfile && wantedLockfileFileExists && files[0] == null) {
throw new PnpmError('BROKEN_LOCKFILE', `The lockfile at "${path.join(opts.lockfileDir, WANTED_LOCKFILE)}" is broken: it is empty`)
}
const sopts = {
autoInstallPeers: opts.autoInstallPeers,
excludeLinksFromLockfile: opts.excludeLinksFromLockfile,

View File

@@ -82,6 +82,25 @@ test('readLockfiles() throws on incompatible lockfile in CI when frozenLockfile
})).rejects.toMatchObject({ code: 'ERR_PNPM_LOCKFILE_BREAKING_CHANGE' })
})
test('readLockfiles() throws on an empty wanted lockfile when frozenLockfile is true', async () => {
const lockfileDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pnpm-get-context-'))
await fs.writeFile(path.join(lockfileDir, 'pnpm-lock.yaml'), '')
await expect(readLockfiles({
autoInstallPeers: true,
excludeLinksFromLockfile: false,
peersSuffixMaxLength: 1000,
ci: false,
force: false,
frozenLockfile: true,
projects: [{ id: '.' as ProjectId, rootDir: lockfileDir as ProjectRootDir }],
lockfileDir,
registry: 'https://registry.npmjs.org/',
useLockfile: true,
internalPnpmDir: path.join(lockfileDir, 'node_modules', '.pnpm'),
})).rejects.toMatchObject({ code: 'ERR_PNPM_BROKEN_LOCKFILE' })
})
test('readLockfiles() ignores incompatible lockfile in CI when frozenLockfile is false', async () => {
const lockfileDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pnpm-get-context-'))
await fs.writeFile(path.join(lockfileDir, 'pnpm-lock.yaml'), 'lockfileVersion: 1.0\nimporters:\n .:\n specifiers: {}\n')

View File

@@ -15,6 +15,9 @@
{
"path": "../../core/core-loggers"
},
{
"path": "../../core/error"
},
{
"path": "../../core/logger"
},

View File

@@ -65,6 +65,38 @@ test('installation fails by default if the lockfile contains a wrong checksum, b
}, testDefaults({ force: true }, { retry: { retries: 0 } }))).rejects.toThrow(/Got unexpected checksum for/)
})
test('an install fails closed when a registry tarball entry in the lockfile is missing its integrity', async () => {
const project = prepareEmpty()
const { updatedManifest: manifest } = await addDependenciesToPackage({},
['is-positive@1.0.0'],
testDefaults()
)
// Simulate a lockfile written by an older pnpm where the registry never
// provided an integrity. A missing integrity is indistinguishable from a
// tampered lockfile, so it must never be silently healed: both frozen and
// non-frozen installs fail closed.
const lockfileWithoutIntegrity = clone(project.readLockfile())
delete (lockfileWithoutIntegrity.packages['is-positive@1.0.0'].resolution as TarballResolution).integrity
writeYamlFileSync(WANTED_LOCKFILE, lockfileWithoutIntegrity, { lineWidth: 1000 })
rimrafSync('node_modules')
await expect(mutateModulesInSingleProject({
manifest,
mutation: 'install',
rootDir: process.cwd() as ProjectRootDir,
}, testDefaults({ frozenLockfile: true }, { retry: { retries: 0 } }))).rejects.toThrow(/has no "integrity" field/)
writeYamlFileSync(WANTED_LOCKFILE, lockfileWithoutIntegrity, { lineWidth: 1000 })
rimrafSync('node_modules')
await expect(mutateModulesInSingleProject({
manifest,
mutation: 'install',
rootDir: process.cwd() as ProjectRootDir,
}, testDefaults({}, { retry: { retries: 0 } }))).rejects.toThrow(/has no "integrity" field/)
})
test('installation fails by default if the lockfile contains the wrong checksum and the store is clean', async () => {
await addDistTag({ package: '@pnpm.e2e/dep-of-pkg-with-1-dep', version: '100.0.0', distTag: 'latest' })
const project = prepareEmpty()

View File

@@ -9,6 +9,8 @@ import type { ResolutionVerifier } from '@pnpm/resolving.resolver-base'
import { verifyLockfileResolutions } from '../../src/install/verifyLockfileResolutions.js'
const GIT_COMMIT = '0123456789abcdef0123456789abcdef01234567'
function makeLockfile (packages: Record<string, { resolution: unknown, version?: string }>): LockfileObject {
return {
lockfileVersion: '9.0',
@@ -83,7 +85,7 @@ test('throws a generic code with per-entry codes in the breakdown when violation
await expect(verifyLockfileResolutions(lockfile, [verifier])).rejects.toMatchObject({
// Mixed-code batch escalates to the generic LOCKFILE_RESOLUTION_VERIFICATION
// code so downstream handlers don't mis-route on whichever entry happened
// code so downstream handlers don't branch on whichever entry happened
// to land first.
code: 'ERR_PNPM_LOCKFILE_RESOLUTION_VERIFICATION',
// Per-entry code is included in the breakdown so the user can see
@@ -362,7 +364,7 @@ test('rejects a registry-style depPath backed by a git resolution, even with no
test('rejects a registry-style depPath backed by a git-hosted tarball resolution', async () => {
const lockfile = makeLockfile({
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: 'https://codeload.github.com/org/foo/tar.gz/abc123', gitHosted: true } },
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: `https://codeload.github.com/org/foo/tar.gz/${GIT_COMMIT}`, gitHosted: true } },
})
await expect(verifyLockfileResolutions(lockfile, [])).rejects.toMatchObject({
code: 'ERR_PNPM_RESOLUTION_SHAPE_MISMATCH',
@@ -424,7 +426,7 @@ test('rejects a registry-style depPath whose git-host tarball clears the gitHost
// dodge a flag-only check. The URL itself must still flag it.
for (const gitHosted of [false, 'true', 'false', 0, 1]) {
const lockfile = makeLockfile({
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: 'https://codeload.github.com/org/foo/tar.gz/abc123', gitHosted } as never },
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: `https://codeload.github.com/org/foo/tar.gz/${GIT_COMMIT}`, gitHosted } as never },
})
// eslint-disable-next-line no-await-in-loop
await expect(verifyLockfileResolutions(lockfile, [])).rejects.toMatchObject({
@@ -474,7 +476,7 @@ test('rejects a registry-style depPath whose git-host tarball varies the host ca
// Hostnames are case-insensitive; an upper-case codeload host paired with
// gitHosted: false must not pass as registry-shaped.
const lockfile = makeLockfile({
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: 'https://CODELOAD.GITHUB.COM/org/foo/tar.gz/abc123', gitHosted: false } as never },
'foo@1.0.0': { resolution: { integrity: 'sha512-deadbeef', tarball: `https://CODELOAD.GITHUB.COM/org/foo/tar.gz/${GIT_COMMIT}`, gitHosted: false } as never },
})
await expect(verifyLockfileResolutions(lockfile, [])).rejects.toMatchObject({
code: 'ERR_PNPM_RESOLUTION_SHAPE_MISMATCH',

View File

@@ -399,6 +399,8 @@ export async function resolveDependencies (
}
}
await waitForResolutionFetches(resolvedPkgsById)
const newLockfile = updateLockfile({
dependenciesGraph,
lockfile: opts.wantedLockfile,
@@ -530,6 +532,22 @@ function alignDependencyTypes (manifest: ProjectManifest, projectSnapshot: Proje
}
}
/**
* Waits for fetches that complete resolution data used by the lockfile snapshot and
* virtual-store paths. Other package fetches are awaited later by `waitTillAllFetchingsFinish`.
*/
async function waitForResolutionFetches (resolvedPkgsById: Record<string, ResolvedPackage>): Promise<void> {
const fetches: Array<Promise<unknown>> = []
for (const pkg of Object.values(resolvedPkgsById)) {
if (pkg.resolutionNeedsFetch && pkg.fetching != null) {
fetches.push(pkg.fetching())
}
}
if (fetches.length > 0) {
await Promise.all(fetches)
}
}
function getAliasToDependencyTypeMap (manifest: ProjectManifest): Record<string, DependenciesField> {
const depTypesOfAliases: Record<string, DependenciesField> = {}
for (const depType of DEPENDENCIES_FIELDS) {

View File

@@ -297,6 +297,12 @@ export interface ResolvedPackage {
dev: boolean
optional: boolean
fetching: () => Promise<PkgRequestFetchResult>
/**
* The resolution can't be completed without awaiting `fetching` (e.g. a registry tarball
* whose integrity is computed from the downloaded bytes). The lockfile snapshot and the
* virtual-store paths derived from the integrity must await `fetching` for these first.
*/
resolutionNeedsFetch?: boolean
filesIndexFile: string
name: string
version: string
@@ -2193,6 +2199,7 @@ function getResolvedPackage (
pkgIdWithPatchHash: options.pkgIdWithPatchHash,
dev: options.wantedDependency.dev,
fetching: options.pkgResponse.fetching!,
resolutionNeedsFetch: options.pkgResponse.resolutionNeedsFetch,
filesIndexFile: options.pkgResponse.filesIndexFile!,
hasBin: options.hasBin,
hasBundledDependencies: !((options.pkg.bundledDependencies ?? options.pkg.bundleDependencies) == null),

View File

@@ -11,12 +11,13 @@ import type {
FetchOptions,
FetchResult,
} from '@pnpm/fetching.fetcher-base'
import { pickFetcher } from '@pnpm/fetching.pick-fetcher'
import { type PickedFetcher, pickFetcher } from '@pnpm/fetching.pick-fetcher'
import gfs from '@pnpm/fs.graceful-fs'
import type { CustomFetcher } from '@pnpm/hooks.types'
import { logger } from '@pnpm/logger'
import {
type AtomicResolution,
classifyResolution,
type DirectoryResolution,
type PlatformAssetResolution,
type PreferredVersions,
@@ -132,6 +133,8 @@ export function createPackageRequester (
requestsQueue,
resolve: opts.resolve,
storeDir: opts.storeDir,
fetchers: opts.fetchers,
customFetchers: opts.customFetchers,
})
return Object.assign(requestPackage, {
@@ -154,6 +157,8 @@ async function resolveAndFetch (
resolve: ResolveFunction
fetchPackageToStore: FetchPackageToStoreFunction
storeDir: string
fetchers: Fetchers
customFetchers?: CustomFetcher[]
},
wantedDependency: WantedDependency & { optional?: boolean },
options: RequestPackageOptions
@@ -261,9 +266,19 @@ async function resolveAndFetch (
})
)
)
// We can skip fetching the package only if the manifest
// is present after resolution AND the content of the package has not changed
if ((options.skipFetch === true || isInstallable === false) && !integrityChanged && (manifest != null)) {
const fetcherForResolution = resolution.type === 'variations'
? undefined
: await pickFetcher(ctx.fetchers, resolution as AtomicResolution, {
customFetchers: ctx.customFetchers,
packageId: id,
})
const resolutionNeedsFetchHook = fetcherForResolution?.resolutionNeedsFetch
const resolutionNeedsFetch = typeof resolutionNeedsFetchHook === 'function'
? resolutionNeedsFetchHook(resolution)
: false
// Fetching can be skipped only when the manifest is available, the package content
// did not change, and the resolution does not need fetch-derived data.
if ((options.skipFetch === true || isInstallable === false) && !resolutionNeedsFetch && !integrityChanged && (manifest != null)) {
return {
body: {
id,
@@ -287,6 +302,8 @@ async function resolveAndFetch (
allowBuild: options.allowBuild,
fetchRawManifest: true,
force: integrityChanged,
populateMissingIntegrity: resolutionNeedsFetch,
pickedFetcher: fetcherForResolution,
ignoreScripts: options.ignoreScripts,
lockfileDir: options.lockfileDir,
pkg: {
@@ -313,11 +330,30 @@ async function resolveAndFetch (
manifest = loadedManifest as unknown as DependencyManifest
}
}
// Add integrity to resolution if it was computed during fetching (only for TarballResolution)
if (fetchedResult.integrity && !resolution.type && !(resolution as TarballResolution).integrity) {
// Add computed integrity to tarball resolutions. `variations` spans multiple
// platform variants, so do not write one machine's integrity into the shared resolution.
if (resolution.type !== 'variations' && fetchedResult.integrity != null && getExpectedIntegrity(resolution) == null) {
(resolution as TarballResolution).integrity = fetchedResult.integrity
}
}
let fetching = fetchResult.fetching
if (resolutionNeedsFetch) {
let populating: Promise<PkgRequestFetchResult> | undefined
fetching = () => {
populating ??= fetchResult.fetching().then((fetchedResult) => {
if (fetchedResult.integrity != null && getExpectedIntegrity(resolution) == null) {
(resolution as TarballResolution).integrity = fetchedResult.integrity
}
return fetchedResult
}).catch((err: unknown) => {
// Cache only fulfilled fetches; rejected fetches may be retried.
populating = undefined
throw err
})
return populating
}
}
// Check installability now that we have the manifest (for git/tarball packages without registry metadata)
if (isInstallable === undefined && manifest != null) {
isInstallable = ctx.force === true || packageIsInstallable(id, manifest, {
@@ -343,8 +379,9 @@ async function resolveAndFetch (
alias,
policyViolation,
},
fetching: fetchResult.fetching,
fetching,
filesIndexFile: fetchResult.filesIndexFile,
resolutionNeedsFetch,
}
}
@@ -406,7 +443,8 @@ function fetchToStore (
fetch: (
packageId: string,
resolution: AtomicResolution,
opts: FetchOptions
opts: FetchOptions,
pickedFetcher?: PickedFetcher
) => Promise<FetchResult>
fetchingLocker: Map<string, FetchLock>
requestsQueue: {
@@ -526,8 +564,16 @@ function fetchToStore (
const isLocalTarballDep = opts.pkg.id.startsWith('file:')
const isLocalPkg = resolution.type === 'directory'
const populateMissingIntegrity = opts.populateMissingIntegrity === true
if (!populateMissingIntegrity) {
assertFetchableResolution(opts.pkg.id, resolution)
}
let refetchingStoredPackage = false
if (
!opts.force &&
!populateMissingIntegrity &&
(
!isLocalTarballDep ||
await tarballIsUpToDate(opts.pkg.resolution as any, target, opts.lockfileDir) // eslint-disable-line
@@ -545,12 +591,14 @@ function fetchToStore (
})
return
}
if ((files?.filesMap) != null) {
packageRequestLogger.warn({
message: `Refetching ${target} to store. It was either modified or had no integrity checksums`,
prefix: opts.lockfileDir,
})
}
refetchingStoredPackage = (files?.filesMap) != null
}
if (refetchingStoredPackage) {
packageRequestLogger.warn({
message: `Refetching ${target} to store. It was either modified or had no integrity checksums`,
prefix: opts.lockfileDir,
})
}
// We fetch into targetStage directory first and then fs.rename() it to the
@@ -589,10 +637,11 @@ function fetchToStore (
name: opts.pkg.name,
version: opts.pkg.version,
},
}
},
opts.pickedFetcher
), { priority })
const integrity = (opts.pkg.resolution as TarballResolution).integrity ?? fetchedPackage.integrity
const integrity = getExpectedIntegrity(opts.pkg.resolution) ?? fetchedPackage.integrity
if (isLocalTarballDep && integrity) {
await fs.mkdir(target, { recursive: true })
await gfs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), integrity, 'utf8')
@@ -618,6 +667,20 @@ async function readBundledManifest (pkgJsonPath: string): Promise<BundledManifes
return normalizeBundledManifest(await loadJsonFile<DependencyManifest>(pkgJsonPath))
}
function getExpectedIntegrity (resolution: unknown): string | undefined {
const integrity = (resolution as { integrity?: unknown }).integrity
return typeof integrity === 'string' && integrity.length > 0
? integrity
: undefined
}
function assertFetchableResolution (depPath: string, resolution: AtomicResolution): void {
if (classifyResolution(resolution) !== 'remoteTarball') return
if (getExpectedIntegrity(resolution) != null) return
throw new PnpmError('MISSING_TARBALL_INTEGRITY',
`Cannot fetch package "${depPath}" from the lockfile: it has no "integrity" field, so the downloaded tarball cannot be verified. Run a fresh install to repair the lockfile.`)
}
async function tarballIsUpToDate (
resolution: {
integrity?: string
@@ -650,11 +713,11 @@ async function fetcher (
customFetchers: CustomFetcher[] | undefined,
packageId: string,
resolution: AtomicResolution,
opts: FetchOptions
opts: FetchOptions,
pickedFetcher?: PickedFetcher
): Promise<FetchResult> {
try {
// pickFetcher now handles custom fetcher hooks internally
const fetch = await pickFetcher(fetcherByHostingType, resolution, {
const fetch = pickedFetcher ?? await pickFetcher(fetcherByHostingType, resolution, {
customFetchers,
packageId,
})

View File

@@ -8,7 +8,7 @@ import { createClient } from '@pnpm/installing.client'
import { createPackageRequester, type PackageResponse } from '@pnpm/installing.package-requester'
import { streamParser } from '@pnpm/logger'
import type { PackageFilesIndex } from '@pnpm/store.cafs'
import type { PkgRequestFetchResult, PkgResolutionId, RequestPackageOptions } from '@pnpm/store.controller-types'
import type { PkgRequestFetchResult, PkgResolutionId, RequestPackageOptions, Resolution } from '@pnpm/store.controller-types'
import { createCafsStore } from '@pnpm/store.create-cafs-store'
import { StoreIndex } from '@pnpm/store.index'
import { fixtures } from '@pnpm/test-fixtures'
@@ -98,6 +98,44 @@ test('request package', async () => {
expect(files.resolvedFrom).toBe('remote')
})
test('a custom fetcher is selected once per request, not re-picked on the fetch path', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
let canFetchCalls = 0
const customFetchers = [{
// Claim every registry tarball; `canFetch` is async (the cost this dedup avoids
// running twice), so count its calls.
canFetch: async (_id: string, resolution: { type?: string, tarball?: string }) => {
canFetchCalls++
return resolution.type == null && typeof resolution.tarball === 'string'
},
// Delegate the actual fetch to the standard remote-tarball fetcher.
fetch: async (cafs: any, resolution: any, opts: any, fetchers: any) => // eslint-disable-line @typescript-eslint/no-explicit-any
fetchers.remoteTarball(cafs, resolution, opts),
}]
const requestPackage = createPackageRequester({
resolve,
fetchers,
customFetchers: customFetchers as never,
cafs,
networkConcurrency: 1,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
const projectDir = temporaryDirectory()
const pkgResponse = await requestPackage({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {
downloadPriority: 0,
lockfileDir: projectDir,
preferredVersions: {},
projectDir,
})
await pkgResponse.fetching!()
expect(canFetchCalls).toBe(1)
})
test('request package but skip fetching', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
@@ -488,6 +526,50 @@ test('integrity of a tarball dependency is preserved when the resolver returns n
expect(response.body.resolution).toStrictEqual({ tarball, integrity })
})
test('computed tarball integrity replaces a non-string lockfile integrity', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
const projectDir = temporaryDirectory()
const resolution = {
integrity: true,
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
} as unknown as Resolution
const customResolve: typeof resolve = async () => ({
id: 'is-positive@1.0.0' as PkgResolutionId,
latest: '1.0.0',
resolution,
manifest: {
name: 'is-positive',
version: '1.0.0',
},
resolvedVia: 'npm-registry',
})
const requestPackage = createPackageRequester({
resolve: customResolve,
fetchers,
cafs,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
const response = await requestPackage({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {
downloadPriority: 0,
lockfileDir: projectDir,
preferredVersions: {},
projectDir,
skipFetch: true,
})
expect(response.fetching).toBeTruthy()
expect((response.body.resolution as { integrity?: unknown }).integrity).toBe(true)
const fetchedResult = await response.fetching!()
expect(fetchedResult.integrity).toMatch(/^sha512-/)
expect((response.body.resolution as { integrity?: unknown }).integrity).toBe(fetchedResult.integrity)
})
test('fetchPackageToStore()', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
@@ -557,6 +639,37 @@ test('fetchPackageToStore()', async () => {
)
})
test('fetchPackageToStore() rejects remote tarballs without integrity by default', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
const packageRequester = createPackageRequester({
resolve,
fetchers: createFetchersForStore(storeDir),
cafs,
networkConcurrency: 1,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
const fetchResult = packageRequester.fetchPackageToStore({
force: false,
lockfileDir: temporaryDirectory(),
pkg: {
name: 'is-positive',
version: '1.0.0',
id: 'is-positive@1.0.0',
resolution: {
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
},
},
})
await expect(fetchResult.fetching()).rejects.toMatchObject({
code: 'ERR_PNPM_MISSING_TARBALL_INTEGRITY',
})
})
test('fetchPackageToStore() concurrency check', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
@@ -784,6 +897,7 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async () => {
}
const fetchResults = await Promise.all([
packageRequester.fetchPackageToStore({
populateMissingIntegrity: true,
fetchRawManifest: false,
force: false,
lockfileDir: temporaryDirectory(),
@@ -795,6 +909,7 @@ test('fetchPackageToStore() fetch raw manifest of cached package', async () => {
},
}),
packageRequester.fetchPackageToStore({
populateMissingIntegrity: true,
fetchRawManifest: true,
force: false,
lockfileDir: temporaryDirectory(),
@@ -817,9 +932,11 @@ test('refetch package to store if it has been modified', async () => {
const localFetchers = createFetchersForStore(storeDir)
const pkgId = 'magic-hook@2.0.0'
const resolution = {
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/magic-hook/-/magic-hook-2.0.0.tgz`,
}
const resolution = (await resolve({ alias: 'magic-hook', bareSpecifier: '2.0.0' }, {
lockfileDir,
preferredVersions: {},
projectDir: lockfileDir,
})).resolution
let indexJsFile!: string
{
@@ -1290,6 +1407,111 @@ test('HTTP tarball without integrity gets integrity computed during fetch', asyn
expect((pkgResponse.body.resolution as { integrity?: string }).integrity).toMatch(/^sha512-/)
})
test('registry tarball without integrity gets integrity computed even when already in the local store', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
const projectDir = temporaryDirectory()
// First, populate the local store with a normal request that has integrity.
const seedRequestPackage = createPackageRequester({
resolve,
fetchers,
cafs,
networkConcurrency: 1,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
const seedResponse = await seedRequestPackage({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {
downloadPriority: 0,
lockfileDir: projectDir,
preferredVersions: {},
projectDir,
}) as PackageResponse & { fetching: () => Promise<PkgRequestFetchResult> }
await seedResponse.fetching()
// Store hits cannot supply missing lockfile integrity; this path must fetch.
const resolveWithoutIntegrity: typeof resolve = async () => ({
id: 'is-positive@1.0.0' as PkgResolutionId,
latest: '1.0.0',
resolution: {
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
},
manifest: {
name: 'is-positive',
version: '1.0.0',
},
resolvedVia: 'npm-registry',
})
const requestPackage = createPackageRequester({
resolve: resolveWithoutIntegrity,
fetchers,
cafs,
networkConcurrency: 1,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
const pkgResponse = await requestPackage({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {
downloadPriority: 0,
lockfileDir: projectDir,
preferredVersions: {},
projectDir,
}) as PackageResponse & { fetching: () => Promise<PkgRequestFetchResult> }
await pkgResponse.fetching()
expect(pkgResponse.body.resolution).toHaveProperty('integrity')
expect((pkgResponse.body.resolution as { integrity?: string }).integrity).toMatch(/^sha512-/)
})
test('skipFetch still downloads the tarball to compute a missing integrity', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)
const projectDir = temporaryDirectory()
const resolveWithoutIntegrity: typeof resolve = async () => ({
id: 'is-positive@1.0.0' as PkgResolutionId,
latest: '1.0.0',
resolution: {
tarball: `http://localhost:${REGISTRY_MOCK_PORT}/is-positive/-/is-positive-1.0.0.tgz`,
},
manifest: {
name: 'is-positive',
version: '1.0.0',
},
resolvedVia: 'npm-registry',
})
const requestPackage = createPackageRequester({
resolve: resolveWithoutIntegrity,
fetchers,
cafs,
networkConcurrency: 1,
storeDir,
verifyStoreIntegrity: true,
virtualStoreDirMaxLength: 120,
})
// Even though `skipFetch` would normally avoid a download, a registry tarball with no
// integrity must be fetched so the integrity can be computed for the lockfile.
const pkgResponse = await requestPackage({ alias: 'is-positive', bareSpecifier: '1.0.0' }, {
downloadPriority: 0,
lockfileDir: projectDir,
preferredVersions: {},
projectDir,
skipFetch: true,
}) as PackageResponse & { fetching: () => Promise<PkgRequestFetchResult> }
// The integrity is populated when the fetch is awaited (the resolver awaits these before
// building the lockfile), not synchronously, so resolution isn't blocked on the download.
expect((pkgResponse.body.resolution as { integrity?: string }).integrity).toBeUndefined()
await pkgResponse.fetching()
expect(pkgResponse.body.resolution).toHaveProperty('integrity')
expect((pkgResponse.body.resolution as { integrity?: string }).integrity).toMatch(/^sha512-/)
})
test('should pass optional flag to resolve function', async () => {
const storeDir = temporaryDirectory()
const cafs = createCafsStore(storeDir)

View File

@@ -5,8 +5,9 @@ export { packageIdFromSnapshot } from './packageIdFromSnapshot.js'
export { packageIsIndependent } from './packageIsIndependent.js'
export { pkgSnapshotToResolution } from './pkgSnapshotToResolution.js'
export { refIsLocalDirectory, refIsLocalTarball } from './refIsLocalTarball.js'
export { isGitHostedTarballUrl, toLockfileResolution } from './toLockfileResolution.js'
export { toLockfileResolution } from './toLockfileResolution.js'
export * from '@pnpm/lockfile.types'
export { isGitHostedTarballUrl } from '@pnpm/resolving.resolver-base'
// for backward compatibility
export const getPkgShortId = refToRelative

View File

@@ -8,7 +8,6 @@ import { getNpmTarballUrl } from '@pnpm/resolving.tarball-url'
import type { Registries } from '@pnpm/types'
import { nameVerFromPkgSnapshot } from './nameVerFromPkgSnapshot.js'
import { isGitHostedTarballUrl } from './toLockfileResolution.js'
export function pkgSnapshotToResolution (
depPath: string,
@@ -16,29 +15,10 @@ export function pkgSnapshotToResolution (
registries: Registries
): Resolution {
const resolution = pkgSnapshot.resolution as TarballResolution
// Tarball-shaped resolutions (no `type` field) must carry `integrity`,
// except where the URL itself anchors the bytes:
// - `file:` tarballs (local file on the user's machine; integrity
// adds nothing the user doesn't already control).
// - Git-hosted tarballs (URL contains the commit SHA; git's content-
// addressed model binds the bytes to the commit). The `gitHosted`
// flag may be absent on legacy lockfiles, so fall back to a URL
// match — same logic as `toLockfileResolution`.
// For any other tarball entry a missing integrity is what a tampered
// lockfile looks like: the worker would mint a fresh integrity from
// whatever bytes the URL returned, so we fail closed here. Pacquet
// enforces the same invariant via
// `pacquet_package_manager::missing_tarball_integrity`.
if (
resolution.type == null &&
resolution.integrity == null &&
!resolution.tarball?.startsWith('file:') &&
!(resolution.gitHosted === true || (resolution.tarball != null && isGitHostedTarballUrl(resolution.tarball)))
) {
throw new PnpmError('MISSING_TARBALL_INTEGRITY',
`Cannot install package "${depPath}": its lockfile entry has no "integrity" field, so pnpm cannot verify the downloaded tarball.`,
{ hint: 'The lockfile may be corrupted or have been tampered with. Restore it from a trusted source, or delete it and re-run installation without --frozen-lockfile to regenerate.' }
)
if (resolution.tarball != null && typeof resolution.tarball !== 'string') {
// Avoid URL string-coercion from malformed YAML lockfile values.
throw new PnpmError('INVALID_TARBALL_RESOLUTION',
`Cannot install package "${depPath}": its lockfile entry has a non-string "tarball" field.`)
}
if (
Boolean(resolution.type) ||
@@ -47,9 +27,8 @@ export function pkgSnapshotToResolution (
) {
return pkgSnapshot.resolution as Resolution
}
// Recover the tarball field for `file:` snapshots whose resolution lost
// its tarball (e.g. lockfiles written by an earlier pnpm 11 version that
// dropped the tarball under `lockfile-include-tarball-url=false`).
// Recover the tarball field for `file:` snapshots whose depPath is the only
// source of the local tarball reference.
const nonSemverVersion = dp.parse(depPath).nonSemverVersion
if (nonSemverVersion?.startsWith('file:')) {
return {
@@ -68,10 +47,10 @@ export function pkgSnapshotToResolution (
registry = registries.default
}
let tarball!: string
if (!(pkgSnapshot.resolution as TarballResolution).tarball) {
if (!resolution.tarball) {
tarball = getTarball(registry)
} else {
tarball = new url.URL((pkgSnapshot.resolution as TarballResolution).tarball,
tarball = new url.URL(resolution.tarball,
registry.endsWith('/') ? registry : `${registry}/`
).toString()
}

View File

@@ -1,5 +1,5 @@
import type { LockfileResolution } from '@pnpm/lockfile.types'
import type { Resolution, TarballResolution } from '@pnpm/resolving.resolver-base'
import { isGitHostedTarballUrl, type Resolution, type TarballResolution } from '@pnpm/resolving.resolver-base'
import { isCanonicalRegistryTarballUrl } from '@pnpm/resolving.tarball-url'
export function toLockfileResolution (
@@ -53,19 +53,3 @@ export function toLockfileResolution (
...(path == null ? {} : { path }),
}
}
// Inlined to avoid pulling @pnpm/fetching.pick-fetcher into the lockfile-utils
// dep graph. Used as a fallback when callers haven't pre-set the
// `gitHosted` field on TarballResolution.
export function isGitHostedTarballUrl (url: string): boolean {
// Schemes and hostnames are case-insensitive, so match against a lowercased
// copy: a tampered `https://CODELOAD.GITHUB.COM/...` must not slip past as a
// non-git-hosted (and therefore registry-trusted) tarball. Only the
// lowercased copy is inspected; the original URL is never rewritten.
const lowerUrl = url.toLowerCase()
return (
lowerUrl.startsWith('https://codeload.github.com/') ||
lowerUrl.startsWith('https://bitbucket.org/') ||
lowerUrl.startsWith('https://gitlab.com/')
) && lowerUrl.includes('tar.gz')
}

View File

@@ -1,6 +1,22 @@
import { expect, test } from '@jest/globals'
import { pkgSnapshotToResolution } from '@pnpm/lockfile.utils'
const GIT_TARBALL = 'https://codeload.github.com/foo/bar/tar.gz/0123456789abcdef0123456789abcdef01234567'
const LEGACY_GIT_TARBALL = 'https://codeload.github.com/kevva/is-negative/tar.gz/0123456789abcdef0123456789abcdef01234567'
test('pkgSnapshotToResolution() fails closed on a non-string tarball', () => {
// A tampered lockfile (YAML) could carry a non-string `tarball` that `new URL()` would
// string-coerce into an attacker-controlled URL.
expect(() => pkgSnapshotToResolution('foo@1.0.0', {
resolution: {
integrity: 'sha512-AAAA',
tarball: ['https://attacker.example/foo.tgz'],
},
} as never, { default: 'https://registry.npmjs.org/' })).toThrow(
expect.objectContaining({ code: 'ERR_PNPM_INVALID_TARBALL_RESOLUTION' })
)
})
test('pkgSnapshotToResolution()', () => {
expect(pkgSnapshotToResolution('foo@1.0.0', {
resolution: {
@@ -54,49 +70,26 @@ test('pkgSnapshotToResolution()', () => {
})
})
test('pkgSnapshotToResolution() rejects a remote tarball resolution that has no integrity', () => {
// A tampered or malformed lockfile that strips the `integrity` field
// would otherwise let pnpm download the URL contents unchecked. The
// helper must fail closed so neither install path nor any read-only
// consumer (sbom, list, etc.) silently trusts the lockfile entry.
expect(() => pkgSnapshotToResolution('foo@1.0.0', {
resolution: {
tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz',
},
}, { default: 'https://registry.npmjs.org/' })).toThrow(expect.objectContaining({ code: 'ERR_PNPM_MISSING_TARBALL_INTEGRITY' }))
// A tarball URL on an arbitrary CDN (no `gitHosted` flag, no known git
// host pattern) is still a regular remote tarball — integrity required.
expect(() => pkgSnapshotToResolution('xlsx@https+++cdn.sheetjs.com+xlsx-0.18.9+xlsx-0.18.9.tgz', {
resolution: {
tarball: 'https://cdn.sheetjs.com/xlsx-0.18.9/xlsx-0.18.9.tgz',
},
}, { default: 'https://registry.npmjs.org/' })).toThrow(expect.objectContaining({ code: 'ERR_PNPM_MISSING_TARBALL_INTEGRITY' }))
})
test('pkgSnapshotToResolution() allows git-hosted and file: tarballs to lack integrity', () => {
// Git-hosted tarballs are anchored by the commit SHA in their URL —
// pnpm's own install pipeline writes them without `integrity:` (see
// the `with-git-protocol-dep` fixture). Both the explicit
// `gitHosted: true` flag and a URL on a known git host must bypass
// the integrity check, matching the URL-fallback logic in
// `toLockfileResolution`.
test('pkgSnapshotToResolution() converts git-hosted and file: tarball snapshots', () => {
// The integrity requirement for registry tarballs is enforced by the npm
// resolver's lockfile verifier, not here; this pure conversion returns
// git-hosted (commit-anchored) and file: (local) tarballs as-is.
expect(pkgSnapshotToResolution('foo@https+++github.com+foo+bar', {
resolution: {
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abc1234',
tarball: GIT_TARBALL,
gitHosted: true,
},
}, { default: 'https://registry.npmjs.org/' })).toEqual({
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abc1234',
tarball: GIT_TARBALL,
gitHosted: true,
})
expect(pkgSnapshotToResolution('is-negative@https+++codeload.github.com+kevva+is-negative+tar.gz+abc', {
resolution: {
tarball: 'https://codeload.github.com/kevva/is-negative/tar.gz/abc1234',
tarball: LEGACY_GIT_TARBALL,
},
}, { default: 'https://registry.npmjs.org/' })).toEqual({
tarball: 'https://codeload.github.com/kevva/is-negative/tar.gz/abc1234',
tarball: LEGACY_GIT_TARBALL,
})
// `file:` tarballs are local files; the user already controls the

View File

@@ -2,6 +2,7 @@ import { expect, test } from '@jest/globals'
import { toLockfileResolution } from '@pnpm/lockfile.utils'
const REGISTRY = 'https://registry.npmjs.org/'
const GIT_TARBALL = 'https://codeload.github.com/foo/bar/tar.gz/0123456789abcdef0123456789abcdef01234567'
test('keeps the tarball when lockfileIncludeTarballUrl is true', () => {
expect(toLockfileResolution(
@@ -95,12 +96,12 @@ test('keeps file: tarballs even when lockfileIncludeTarballUrl is undefined', ()
test('keeps git-hosted tarballs when lockfileIncludeTarballUrl is false', () => {
expect(toLockfileResolution(
{ name: 'foo', version: '1.0.0' },
{ integrity: 'sha512-AAAA', tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef' },
{ integrity: 'sha512-AAAA', tarball: GIT_TARBALL },
REGISTRY,
false
)).toEqual({
integrity: 'sha512-AAAA',
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef',
tarball: GIT_TARBALL,
gitHosted: true,
})
})
@@ -111,12 +112,12 @@ test('keeps the path of a git-hosted tarball pointing to a subdirectory', () =>
// unpack the repository root. See https://github.com/pnpm/pnpm/issues/12304.
expect(toLockfileResolution(
{ name: 'foo', version: '1.0.0' },
{ integrity: 'sha512-AAAA', tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef', gitHosted: true, path: '/packages/foo' },
{ integrity: 'sha512-AAAA', tarball: GIT_TARBALL, gitHosted: true, path: '/packages/foo' },
REGISTRY,
false
)).toEqual({
integrity: 'sha512-AAAA',
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef',
tarball: GIT_TARBALL,
gitHosted: true,
path: '/packages/foo',
})
@@ -125,12 +126,12 @@ test('keeps the path of a git-hosted tarball pointing to a subdirectory', () =>
test('keeps the path of a git-hosted tarball when lockfileIncludeTarballUrl is true', () => {
expect(toLockfileResolution(
{ name: 'foo', version: '1.0.0' },
{ integrity: 'sha512-AAAA', tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef', gitHosted: true, path: '/packages/foo' },
{ integrity: 'sha512-AAAA', tarball: GIT_TARBALL, gitHosted: true, path: '/packages/foo' },
REGISTRY,
true
)).toEqual({
integrity: 'sha512-AAAA',
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef',
tarball: GIT_TARBALL,
gitHosted: true,
path: '/packages/foo',
})
@@ -139,12 +140,12 @@ test('keeps the path of a git-hosted tarball when lockfileIncludeTarballUrl is t
test('records gitHosted on the lockfile entry when set on the resolution', () => {
expect(toLockfileResolution(
{ name: 'foo', version: '1.0.0' },
{ integrity: 'sha512-AAAA', tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef', gitHosted: true },
{ integrity: 'sha512-AAAA', tarball: GIT_TARBALL, gitHosted: true },
REGISTRY,
true
)).toEqual({
integrity: 'sha512-AAAA',
tarball: 'https://codeload.github.com/foo/bar/tar.gz/abcdef',
tarball: GIT_TARBALL,
gitHosted: true,
})
})

View File

@@ -40,7 +40,6 @@
"@pnpm/constants": "workspace:*",
"@pnpm/crypto.hash": "workspace:*",
"@pnpm/error": "workspace:*",
"@pnpm/fetching.pick-fetcher": "workspace:*",
"@pnpm/fs.packlist": "workspace:*",
"@pnpm/installing.commands": "workspace:*",
"@pnpm/installing.modules-yaml": "workspace:*",

View File

@@ -3,9 +3,8 @@ import path from 'node:path'
import { confirm, select } from '@inquirer/prompts'
import type { Config } from '@pnpm/config.reader'
import { PnpmError } from '@pnpm/error'
import { isGitHostedPkgUrl } from '@pnpm/fetching.pick-fetcher'
import { readCurrentLockfile, type TarballResolution } from '@pnpm/lockfile.fs'
import { nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils'
import { isGitHostedTarballUrl, nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils'
import { parseWantedDependency, type ParseWantedDependencyResult } from '@pnpm/resolving.parse-wanted-dependency'
import { realpathMissing } from 'realpath-missing'
import semver from 'semver'
@@ -117,7 +116,7 @@ export async function getVersionsFromLockfile (dep: ParseWantedDependencyResult,
const tarball = (pkgSnapshot.resolution as TarballResolution)?.tarball ?? ''
return {
...nameVerFromPkgSnapshot(depPath, pkgSnapshot),
gitTarballUrl: (isGitHostedPkgUrl(tarball) || isPkgPrNewUrl(tarball)) ? tarball : undefined,
gitTarballUrl: (isGitHostedTarballUrl(tarball) || isPkgPrNewUrl(tarball)) ? tarball : undefined,
}
})
.filter(({ name }) => name === pkgName)

View File

@@ -39,9 +39,6 @@
{
"path": "../../crypto/hash"
},
{
"path": "../../fetching/pick-fetcher"
},
{
"path": "../../fs/packlist"
},

View File

@@ -4,9 +4,10 @@ import { FULL_META_DIR } from '@pnpm/constants'
import { PnpmError } from '@pnpm/error'
import type { GetAuthHeader } from '@pnpm/fetching.types'
import type { PackageInRegistry, PackageMeta } from '@pnpm/resolving.registry.types'
import type {
Resolution,
ResolutionVerifier,
import {
isGitHostedTarballUrl,
type Resolution,
type ResolutionVerifier,
} from '@pnpm/resolving.resolver-base'
import type { PackageVersionPolicy, Registries, TrustPolicy } from '@pnpm/types'
import semver from 'semver'
@@ -25,6 +26,7 @@ import { getPkgMetaCacheKey, getPkgMirrorPath, loadMeta, warnMissingTimeFieldOnc
import { failIfTrustDowngraded } from './trustChecks.js'
import {
MINIMUM_RELEASE_AGE_VIOLATION_CODE,
MISSING_TARBALL_INTEGRITY_VIOLATION_CODE,
TARBALL_URL_MISMATCH_VIOLATION_CODE,
TRUST_DOWNGRADE_VIOLATION_CODE,
} from './violationCodes.js'
@@ -178,16 +180,42 @@ export function createNpmResolutionVerifier (
const trustPolicyIgnoreAfter = opts.trustPolicyIgnoreAfter
const verify: ResolutionVerifier['verify'] = async (resolution, { name, version, nonSemverVersion }) => {
if (!isNpmRegistryResolution(resolution)) return { ok: true }
if (!isRegistryTarballResolution(resolution)) return { ok: true }
// Network-free structural checks must run before registry metadata shortcuts.
const integrity = (resolution as { integrity?: unknown }).integrity
if (typeof integrity !== 'string' || integrity.length === 0) {
return {
ok: false,
code: MISSING_TARBALL_INTEGRITY_VIOLATION_CODE,
reason: 'has no "integrity" field, so its downloaded tarball cannot be verified',
}
}
// URL/git-keyed entries are deliberate non-registry deps. They can still
// carry a semver `version` copied from the resolved manifest, so the
// semver guard below isn't enough on its own — the registry policies and
// the tarball-URL binding don't apply to them, and a registry lookup
// would 404.
if (nonSemverVersion != null) return { ok: true }
if (!semver.valid(version)) return { ok: true }
const tarballUrl = (resolution as { tarball?: string }).tarball
if (!semver.valid(version)) {
return {
ok: false,
code: TARBALL_URL_MISMATCH_VIOLATION_CODE,
reason: `has a non-semver version ("${version}") and so cannot be verified against the registry's published metadata`,
}
}
const rawTarball = (resolution as { tarball?: unknown }).tarball
if (rawTarball != null && typeof rawTarball !== 'string') {
return {
ok: false,
code: TARBALL_URL_MISMATCH_VIOLATION_CODE,
reason: 'has a non-string "tarball" field, so its URL cannot be verified',
}
}
const tarballUrl = typeof rawTarball === 'string' ? rawTarball : undefined
const registry = pickRegistryForVersion(opts.registries, namedRegistryPrefixes, name, tarballUrl)
// A registry entry that pins an explicit tarball URL must point at the
@@ -242,6 +270,8 @@ export function createNpmResolutionVerifier (
// applies the binding — otherwise an upgrade could keep trusting a
// lockfile that was only ever age/trust-checked.
tarballUrlBinding: true,
// Same cache identity rule for the missing-integrity structural check.
integrityRequired: true,
minimumReleaseAge,
minimumReleaseAgeExclude: sortedMinAgeExcludes,
trustPolicy: trustPolicy ?? null,
@@ -253,6 +283,10 @@ export function createNpmResolutionVerifier (
// didn't record it can't be trusted to have enforced it.
if (cached.tarballUrlBinding !== true) return false
// The missing-integrity check is also unconditional; older cache records
// without the flag cannot prove they rejected unverifiable tarballs.
if (cached.integrityRequired !== true) return false
// Maturity: a previously cached run under a larger cutoff
// (stricter window) is trustworthy under a smaller current one —
// its set of accepted versions is a subset of today's. The
@@ -914,20 +948,21 @@ function isExcluded (policy: PackageVersionPolicy | undefined, name: string, ver
return false
}
function isNpmRegistryResolution (resolution: Resolution | unknown): boolean {
function isRegistryTarballResolution (resolution: Resolution | unknown): boolean {
if (resolution == null || typeof resolution !== 'object') return false
// Only plain tarball resolutions (npm registry / named registries) have no
// `type` field. Git / directory / binary / custom resolutions all carry one.
if ('type' in resolution && (resolution as { type?: unknown }).type != null) return false
// Git-hosted tarballs (codeload/gitlab/bitbucket) are special-cased in
// the resolver and aren't subject to release-age policy.
if ('gitHosted' in resolution && (resolution as { gitHosted?: boolean }).gitHosted) return false
const tarball = (resolution as { tarball?: unknown }).tarball
if (typeof tarball === 'string') {
// Git-hosted tarballs (codeload/gitlab/bitbucket) are special-cased in
// the resolver and aren't subject to registry policy.
if (isGitHostedTarballUrl(tarball)) return false
// Local/non-registry tarballs (for example `file:`) have no packument
// metadata, so minimumReleaseAge/trustPolicy verification cannot apply.
const protocol = tryParseUrl(tarball)?.protocol
if (protocol != null && protocol !== 'http:' && protocol !== 'https:') return false
}
return 'tarball' in resolution || 'integrity' in resolution
// Canonical registry entries may omit both `tarball` and `integrity`.
return true
}

View File

@@ -11,3 +11,4 @@
export const MINIMUM_RELEASE_AGE_VIOLATION_CODE = 'MINIMUM_RELEASE_AGE_VIOLATION'
export const TRUST_DOWNGRADE_VIOLATION_CODE = 'TRUST_DOWNGRADE'
export const TARBALL_URL_MISMATCH_VIOLATION_CODE = 'TARBALL_URL_MISMATCH'
export const MISSING_TARBALL_INTEGRITY_VIOLATION_CODE = 'MISSING_TARBALL_INTEGRITY'

View File

@@ -300,6 +300,120 @@ test('createNpmResolutionVerifier() skips file: tarball resolutions', async () =
expect(result).toEqual({ ok: true })
})
const REGISTRY_TARBALL = 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz'
test('createNpmResolutionVerifier() rejects a registry tarball with no integrity', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ tarball: REGISTRY_TARBALL } as unknown as Resolution,
{ name: 'foo', version: '1.0.0' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() rejects a canonical registry entry stripped down to {}', async () => {
// A tampered lockfile can delete both the tarball URL and integrity from a canonical
// registry entry; the URL is reconstructed from name+version, so it must still be rejected.
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify({} as unknown as Resolution, { name: 'foo', version: '1.0.0' })
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() treats an empty-string integrity as missing', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ integrity: '', tarball: REGISTRY_TARBALL } as unknown as Resolution,
{ name: 'foo', version: '1.0.0' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() treats a non-string integrity as missing', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
for (const integrity of [true, [], {}] as unknown[]) {
// eslint-disable-next-line no-await-in-loop
const result = await verifier.verify(
{ integrity, tarball: REGISTRY_TARBALL } as unknown as Resolution,
{ name: 'foo', version: '1.0.0' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
}
})
test('createNpmResolutionVerifier() enforces missing integrity even with a non-semver version', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ tarball: REGISTRY_TARBALL } as unknown as Resolution,
{ name: 'foo', version: 'not-a-semver' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() exempts a git-hosted tarball URL recorded without the gitHosted flag', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ tarball: 'https://codeload.github.com/kevva/is-negative/tar.gz/0123456789abcdef0123456789abcdef01234567' } as unknown as Resolution,
{ name: 'is-negative', version: '1.0.0', nonSemverVersion: 'https+++github.com+kevva+is-negative' }
)
expect(result).toStrictEqual({ ok: true })
})
test('createNpmResolutionVerifier() rejects git-host archive URLs that are not pinned to a commit', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ tarball: 'https://codeload.github.com/kevva/is-negative/tar.gz/main' } as unknown as Resolution,
{ name: 'is-negative', version: '1.0.0', nonSemverVersion: 'https+++github.com+kevva+is-negative' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() rejects a forged gitHosted flag on a non-git-hosted tarball', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ gitHosted: true, tarball: 'https://attacker.example/evil-1.0.0.tgz' } as unknown as Resolution,
{ name: 'evil', version: '1.0.0', nonSemverVersion: 'https+++attacker.example+evil' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() enforces missing integrity on a URL-keyed (nonSemverVersion) tarball', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ tarball: 'https://cdn.example/foo/-/foo-1.0.0.tgz' } as unknown as Resolution,
{ name: 'foo', version: '1.0.0', nonSemverVersion: 'https://cdn.example/foo/-/foo-1.0.0.tgz' }
)
expect(result).toMatchObject({ ok: false, code: 'MISSING_TARBALL_INTEGRITY' })
})
test('createNpmResolutionVerifier() passes a URL-keyed tarball that carries integrity without a registry lookup', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ integrity: FAKE_INTEGRITY, tarball: 'https://cdn.example/foo/-/foo-1.0.0.tgz' } as unknown as Resolution,
{ name: 'foo', version: '1.0.0', nonSemverVersion: 'https://cdn.example/foo/-/foo-1.0.0.tgz' }
)
expect(result).toStrictEqual({ ok: true })
})
test('createNpmResolutionVerifier() fails closed on a non-semver version for a registry tarball', async () => {
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ integrity: FAKE_INTEGRITY, tarball: REGISTRY_TARBALL } as unknown as Resolution,
{ name: 'foo', version: 'not-a-semver' }
)
expect(result).toMatchObject({ ok: false, code: 'TARBALL_URL_MISMATCH' })
})
test('createNpmResolutionVerifier() rejects a non-string tarball instead of crashing', async () => {
// A YAML array `tarball` would otherwise be string-coerced into an attacker URL later;
// the verifier fails closed rather than silently skipping the URL-binding check.
const verifier = createNpmResolutionVerifier(makeVerifierOpts())
const result = await verifier.verify(
{ integrity: FAKE_INTEGRITY, tarball: ['https://attacker.example/foo-1.0.0.tgz'] } as unknown as Resolution,
{ name: 'foo', version: '1.0.0' }
)
expect(result).toMatchObject({ ok: false, code: 'TARBALL_URL_MISMATCH' })
})
const FAKE_INTEGRITY = 'sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=='
test('createNpmResolutionVerifier() flags a lockfile tarball URL that does not match the registry metadata', async () => {
@@ -431,6 +545,7 @@ test('createNpmResolutionVerifier() canTrustPastCheck rejects when the trust-exc
// Same policy → trust.
expect(verifier.canTrustPastCheck({
tarballUrlBinding: true,
integrityRequired: true,
minimumReleaseAge: 0,
minimumReleaseAgeExclude: [],
trustPolicy: 'no-downgrade',
@@ -440,6 +555,7 @@ test('createNpmResolutionVerifier() canTrustPastCheck rejects when the trust-exc
// Cached run had a wider exclude list (today's is stricter) → invalidate.
expect(verifier.canTrustPastCheck({
tarballUrlBinding: true,
integrityRequired: true,
minimumReleaseAge: 0,
minimumReleaseAgeExclude: [],
trustPolicy: 'no-downgrade',

View File

@@ -26,8 +26,9 @@
],
"scripts": {
"start": "tsgo --watch",
"lint": "eslint \"src/**/*.ts\"",
"test": "pn compile",
"lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
"test": "pn compile && pn .test",
".test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest",
"prepublishOnly": "tsgo --build",
"compile": "tsgo --build && pn lint --fix"
},
@@ -35,6 +36,7 @@
"@pnpm/types": "workspace:*"
},
"devDependencies": {
"@jest/globals": "catalog:",
"@pnpm/resolving.resolver-base": "workspace:*"
},
"engines": {

View File

@@ -83,6 +83,105 @@ export interface VariationsResolution {
export type Resolution = AtomicResolution | VariationsResolution
const GIT_COMMIT_SHA = /^[0-9a-f]{40}$/i
/**
* A tarball URL is git-hosted when it points at a known git provider's immutable
* archive endpoint. The result gates integrity exemptions, so the match is
* limited to provider-specific path shapes whose ref is a full commit SHA.
*/
export function isGitHostedTarballUrl (url: string): boolean {
if (typeof url !== 'string') return false
let parsedUrl: URL
try {
parsedUrl = new URL(url)
} catch {
return false
}
if (parsedUrl.protocol !== 'https:') return false
switch (parsedUrl.hostname.toLowerCase()) {
case 'codeload.github.com':
return isGitHubCodeloadArchive(parsedUrl)
case 'bitbucket.org':
return isBitbucketArchive(parsedUrl)
case 'gitlab.com':
return isGitLabArchive(parsedUrl)
default:
return false
}
}
function isGitHubCodeloadArchive (url: URL): boolean {
const segments = getPathSegments(url)
return segments.length === 4 && segments[2] === 'tar.gz' && GIT_COMMIT_SHA.test(segments[3])
}
function isBitbucketArchive (url: URL): boolean {
const segments = getPathSegments(url)
if (segments.length !== 4 || segments[2] !== 'get' || !segments[3].endsWith('.tar.gz')) return false
return GIT_COMMIT_SHA.test(segments[3].slice(0, -'.tar.gz'.length))
}
function isGitLabArchive (url: URL): boolean {
const segments = getPathSegments(url)
if (segments.length === 6 &&
segments[0] === 'api' &&
segments[1] === 'v4' &&
segments[2] === 'projects' &&
segments[4] === 'repository' &&
segments[5] === 'archive.tar.gz') {
return GIT_COMMIT_SHA.test(url.searchParams.get('ref') ?? '')
}
const archiveMarkerIndex = segments.findIndex((segment, index) =>
segment === '-' && segments[index + 1] === 'archive'
)
if (archiveMarkerIndex < 2) return false
const ref = segments[archiveMarkerIndex + 2]
const archiveName = segments[archiveMarkerIndex + 3]
return segments.length === archiveMarkerIndex + 4 &&
archiveName?.endsWith('.tar.gz') === true &&
GIT_COMMIT_SHA.test(ref)
}
function getPathSegments (url: URL): string[] {
return url.pathname.split('/').filter(Boolean)
}
export type ResolutionKind =
| 'localTarball'
| 'gitHostedTarball'
| 'remoteTarball'
| 'directory'
| 'git'
| 'binary'
| 'custom'
/**
* Classifies a resolution for fetcher selection. Lockfile-provided flags are
* treated as hints; integrity exemptions depend on the resolved source shape.
*/
export function classifyResolution (resolution: Resolution): ResolutionKind {
if (resolution.type == null) {
const tarball = typeof (resolution as { tarball?: unknown }).tarball === 'string'
? (resolution as { tarball: string }).tarball
: undefined
if (tarball?.startsWith('file:')) return 'localTarball'
if (tarball != null && isGitHostedTarballUrl(tarball)) {
return 'gitHostedTarball'
}
return 'remoteTarball'
}
switch (resolution.type) {
case 'directory':
case 'git':
case 'binary':
return resolution.type
default:
return 'custom'
}
}
/**
* Outcome of asking a `ResolutionVerifier` whether a (name, version,
* resolution) entry from a lockfile is acceptable under whatever policies

View File

@@ -0,0 +1,60 @@
import { expect, test } from '@jest/globals'
import { classifyResolution, isGitHostedTarballUrl, type Resolution } from '@pnpm/resolving.resolver-base'
const GIT_COMMIT = '0123456789abcdef0123456789abcdef01234567'
test('classifyResolution() classifies tarball-shaped resolutions', () => {
expect(classifyResolution({ tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz' } as Resolution)).toBe('remoteTarball')
expect(classifyResolution({ tarball: 'file:foo-1.0.0.tgz' } as Resolution)).toBe('localTarball')
expect(classifyResolution({ tarball: `https://codeload.github.com/foo/bar/tar.gz/${GIT_COMMIT}` } as Resolution)).toBe('gitHostedTarball')
// A recognized git-host URL is git-hosted with or without the `gitHosted` flag.
expect(classifyResolution({ tarball: `https://codeload.github.com/foo/bar/tar.gz/${GIT_COMMIT}`, gitHosted: true } as Resolution)).toBe('gitHostedTarball')
// A forged `gitHosted: true` on a non-git-host URL is NOT trusted (lockfiles are untrusted
// input): it stays a `remoteTarball` so the missing-integrity gate still applies.
expect(classifyResolution({ tarball: 'https://example.com/foo.tgz', gitHosted: true } as Resolution)).toBe('remoteTarball')
})
test('classifyResolution() treats a canonical entry with no tarball URL as a remote tarball', () => {
// A canonical registry entry omits the URL (reconstructed from name+version), so an empty
// resolution must still classify as a remote tarball rather than something exempt.
expect(classifyResolution({} as Resolution)).toBe('remoteTarball')
expect(classifyResolution({ integrity: 'sha512-x' } as Resolution)).toBe('remoteTarball')
})
test('classifyResolution() classifies typed resolutions', () => {
expect(classifyResolution({ type: 'directory', directory: '/foo' } as Resolution)).toBe('directory')
expect(classifyResolution({ type: 'git', repo: 'r', commit: 'c' } as Resolution)).toBe('git')
expect(classifyResolution({ type: 'binary', url: 'u', integrity: 'i', archive: 'tarball', bin: 'b' } as Resolution)).toBe('binary')
expect(classifyResolution({ type: 'variations', variants: [] } as Resolution)).toBe('custom')
expect(classifyResolution({ type: 'custom:cdn' } as Resolution)).toBe('custom')
})
test('classifyResolution() treats a YAML `type: null` as a tarball, not custom', () => {
expect(classifyResolution({ type: null, tarball: 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz' } as unknown as Resolution)).toBe('remoteTarball')
})
test('classifyResolution() does not crash on a non-string tarball from a tampered lockfile', () => {
expect(classifyResolution({ tarball: ['https://attacker.example/foo.tgz'] } as unknown as Resolution)).toBe('remoteTarball')
expect(classifyResolution({ tarball: 42 } as unknown as Resolution)).toBe('remoteTarball')
})
test('isGitHostedTarballUrl() recognizes git provider archive URLs (case-insensitive)', () => {
expect(isGitHostedTarballUrl(`https://codeload.github.com/foo/bar/tar.gz/${GIT_COMMIT}`)).toBe(true)
expect(isGitHostedTarballUrl(`https://gitlab.com/foo/bar/-/archive/${GIT_COMMIT}/bar-${GIT_COMMIT}.tar.gz`)).toBe(true)
expect(isGitHostedTarballUrl(`https://gitlab.com/api/v4/projects/foo%2Fbar/repository/archive.tar.gz?ref=${GIT_COMMIT}`)).toBe(true)
expect(isGitHostedTarballUrl(`https://bitbucket.org/foo/bar/get/${GIT_COMMIT}.tar.gz`)).toBe(true)
// A tampered upper-cased host must not slip past as a registry-trusted tarball.
expect(isGitHostedTarballUrl(`https://CODELOAD.GITHUB.COM/foo/bar/tar.gz/${GIT_COMMIT}`)).toBe(true)
})
test('isGitHostedTarballUrl() rejects non-git-host and non-string inputs', () => {
expect(isGitHostedTarballUrl('https://registry.npmjs.org/foo/-/foo-1.0.0.tgz')).toBe(false)
expect(isGitHostedTarballUrl('https://github.com/foo/bar')).toBe(false)
expect(isGitHostedTarballUrl('https://gitlab.com/foo/bar?download=tar.gz')).toBe(false)
expect(isGitHostedTarballUrl('https://codeload.github.com/foo/bar/tar.gz/main')).toBe(false)
expect(isGitHostedTarballUrl('https://gitlab.com/foo/bar/-/archive/main/bar-main.tar.gz')).toBe(false)
expect(isGitHostedTarballUrl('https://gitlab.com/api/v4/projects/foo%2Fbar/repository/archive.tar.gz')).toBe(false)
expect(isGitHostedTarballUrl('https://bitbucket.org/foo/bar/get/main.tar.gz')).toBe(false)
expect(isGitHostedTarballUrl(undefined as unknown as string)).toBe(false)
expect(isGitHostedTarballUrl(['https://codeload.github.com/x/tar.gz'] as unknown as string)).toBe(false)
})

View File

@@ -0,0 +1,18 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"noEmit": false,
"outDir": "../node_modules/.test.lib",
"rootDir": "..",
"isolatedModules": true
},
"include": [
"**/*.ts",
"../../../__typings__/**/*.d.ts"
],
"references": [
{
"path": ".."
}
]
}

View File

@@ -1,3 +1,9 @@
import type {
BinaryFetcher,
DirectoryFetcher,
FetchFunction,
GitFetcher,
} from '@pnpm/fetching.fetcher-base'
import type {
DirectoryResolution,
PkgResolutionId,
@@ -63,6 +69,8 @@ export type FetchPackageToStoreFunction = (opts: FetchPackageToStoreOptions) =>
export type FetchPackageToStoreFunctionAsync = (opts: FetchPackageToStoreOptions) => Promise<FetchResponse>
type SelectedFetcher = FetchFunction | DirectoryFetcher | GitFetcher | BinaryFetcher
export type GetFilesIndexFilePath = (opts: Pick<FetchPackageToStoreOptions, 'pkg' | 'ignoreScripts'>) => {
filesIndexFile: string
target: string
@@ -77,6 +85,17 @@ export interface FetchPackageToStoreOptions {
allowBuild?: AllowBuild
fetchRawManifest?: boolean
force: boolean
/**
* The resolution can't be completed without a fresh download (e.g. a registry tarball
* whose integrity must be computed from the bytes), so the store copy must not be
* reused. Determined by the fetcher's `resolutionNeedsFetch`.
*/
populateMissingIntegrity?: boolean
/**
* In-process callers may pass the fetcher they already selected for this resolution.
* Omitted when the fetcher has to be selected at fetch time, such as `variations`.
*/
pickedFetcher?: SelectedFetcher
ignoreScripts?: boolean
lockfileDir: string
pkg: PkgNameVersion & {
@@ -139,6 +158,13 @@ export type BundledManifestFunction = () => Promise<BundledManifest | undefined>
export interface PackageResponse {
fetching?: () => Promise<PkgRequestFetchResult>
filesIndexFile?: string
/**
* The resolution can't be completed without awaiting `fetching` — e.g. a registry
* tarball whose integrity is computed from the downloaded bytes. Set by the fetcher's
* `resolutionNeedsFetch`. Callers that read the resolution before fetching (the lockfile
* snapshot, virtual-store paths) must await `fetching` first for these.
*/
resolutionNeedsFetch?: boolean
body: {
isLocal: boolean
isInstallable?: boolean

View File

@@ -170,9 +170,14 @@ fn osv_checkable_tarball_does_not_trust_git_hosted_flag_or_strict_url_parsing()
"https://registry.npmjs.org/foo/-/foo 1.0.0.tgz",
None,
)));
// Mutable git-host archive refs are still checked.
assert!(super::is_osv_checkable_resolution(&tarball(
"https://codeload.github.com/foo/bar/tar.gz/abc123",
Some(false),
)));
// Genuinely git-hosted-by-URL tarballs are skipped regardless of the flag.
assert!(!super::is_osv_checkable_resolution(&tarball(
"https://codeload.github.com/foo/bar/tar.gz/abc123",
"https://codeload.github.com/foo/bar/tar.gz/0123456789abcdef0123456789abcdef01234567",
Some(false),
)));
// Non-http schemes are skipped.