feat(pnpr): rewrite dist.tarball URLs on .tgz registries to <name>@<version> (#17340)

Rewrite served `dist.tarball` URLs on `.tgz` registries to
`{public_url}/{pkg}@{version}` for every addressable version (matching
tar registries), so `serverType: pnpr` drops canonical tarball URLs from
`pnpm-lock.yaml` on both `.tgz` and tar registries while keeping
`{public_url}/{pkg}/-/{filename}` served for existing lockfiles and
older clients.

- Emit `{public_url}/{pkg}@{version}` in `pnpr-upstream` for all
  addressable versions and normalize `<pkg>@{version}` tarball URLs to
  `<scopeless-name>-<version>.tgz` in `tarball_basename_for_package` for
  chained `pnpr` instances.
- Accept round-tripped `{public_url}/{pkg}@{version}` URLs in
  `check_tarball_immutable` during `PUT /:pkg/-rev/:rev` and restore the
  stored `.tgz` URL on disk.
- Route `DELETE /:pkg@:version/-rev/:rev` and
  `DELETE /@:scope/:pkg@:version/-rev/:rev` to `unpublish_version_tarball`
  while rejecting reserved `+r<digits>` revision suffixes.
- Fall back to the version parsed from a canonical `<name>-<version>.tgz`
  request in `expected_tarball_dist` so chained `pnpr` proxies resolve
  non-canonical upstream filenames, and preserve `<name>@<version>`
  routes in `TarballRouter`.

Related to pnpm/tasks#208.
This commit is contained in:
Zoltan Kochan authored and GitHub committed 2026-10-25 17:45:36 +01:00
1 parent 1860c1c75e
commit 23a3f8917a
42 files changed
+802 -245

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
---
"@pnpm/pnpr": patch
---
Served `dist.tarball` URLs on `.tgz` registries now use the `<name>@<version>` route. The `/-/<filename>` route remains supported for existing lockfiles and older clients.
@@ -18,7 +18,7 @@ use std::{
};
fn pacquet_at(workspace: &Path) -> Command {
Command::cargo_bin("pnpm").expect("find the pnpm binary").with_current_dir(workspace)
_utils::pacquet_in(workspace).with_env("PNPM_CONFIG_NPMRC_AUTH_FILE", workspace.join(".npmrc"))
}
/// `pacquet install` resolves the `configDependencies` declared in
@@ -525,7 +525,7 @@ fn add_config_keeps_an_integrity_pin() {
let integrity = lockfile
.lines()
.find_map(|line| line.trim().strip_prefix("resolution: {integrity: "))
.and_then(|rest| rest.strip_suffix('}'))
.and_then(|rest| rest.split([',', '}']).next())
.expect("the lockfile records an integrity");
fs::remove_file(workspace.join("pnpm-lock.yaml")).expect("remove lockfile");
let yaml_path = workspace.join("pnpm-workspace.yaml");
@@ -11,6 +11,7 @@ use pnpm_testing_utils::{
registry::TestRegistry,
};
use std::{
fmt::Write as _,
fs,
path::{Path, PathBuf},
process::Command,
@@ -30,6 +31,15 @@ fn write_manifest(workspace: &Path) {
fs::write(workspace.join("package.json"), manifest.to_string()).expect("write package.json");
}
fn declare_pnpr_registry(workspace: &Path, registry_url: &str) {
let yaml_path = workspace.join("pnpm-workspace.yaml");
let mut workspace_yaml = fs::read_to_string(&yaml_path).expect("read workspace yaml");
write!(workspace_yaml, "\nregistries:\n '{registry_url}':\n serverType: pnpr\n").expect(
"append registry declaration",
);
fs::write(&yaml_path, workspace_yaml).expect("write workspace yaml");
}
/// A resolver that claims `@pnpm.e2e/dep-of-pkg-with-1-dep` and resolves
/// it to `100.1.0` — a version the built-in npm resolver would never pick
/// for the exact `100.0.0` specifier, so its presence in the result
@@ -267,6 +277,7 @@ fn custom_resolver_receives_current_pkg_on_subsequent_installs() {
let AddMockedRegistry { mock_instance, .. } = npmrc_info;
write_manifest(&workspace);
declare_pnpr_registry(&workspace, mock_instance.url());
// First install: the resolver claims nothing, so the npm resolver
// pins 100.0.0 and the lockfile compacts it to `{integrity}`.
fs::write(
@@ -322,10 +333,7 @@ module.exports = {
// the tarball URL re-derived from the registry, like pnpm's
// `pkgSnapshotToResolution`.
let tarball = current_pkg["resolution"]["tarball"].as_str().expect("derived tarball URL");
assert!(
tarball.ends_with("/@pnpm.e2e/dep-of-pkg-with-1-dep/-/dep-of-pkg-with-1-dep-100.0.0.tgz"),
"got: {tarball}",
);
assert!(tarball.ends_with("/@pnpm.e2e/dep-of-pkg-with-1-dep@100.0.0"), "got: {tarball}");
assert!(
current_pkg["resolution"]["integrity"].is_string(),
"the recorded integrity carries over: {current_pkg}",
@@ -443,6 +451,7 @@ fn custom_typed_resolver_without_manifest_installs_dependencies() {
fs::write(workspace.join("vendor/package.tgz"), &body).unwrap();
fs::write(workspace.join("package.json"), r#"{"dependencies":{"custom-typed":"1.0.0"}}"#)
.unwrap();
declare_pnpr_registry(&workspace, mock_instance.url());
let integrity = pnpm_testing_utils::fixtures::sha512_integrity(&body);
let resolution = serde_json::json!({
"type": "custom:vendored", "name": "custom-typed", "version": "1.0.0",
@@ -15,9 +15,8 @@ use std::{collections::HashMap, fs, io};
const CASE_TABLE: &str =
include_str!("../../../../../pnpr/.fixtures/request-logs/metadata-three-installs.json");
const PKG_TARBALL: &str = "GET /@pnpm.e2e/pkg-with-1-dep/-/pkg-with-1-dep-100.1.0.tgz 200";
const DEP_TARBALL: &str =
"GET /@pnpm.e2e/dep-of-pkg-with-1-dep/-/dep-of-pkg-with-1-dep-100.1.0.tgz 200";
const PKG_TARBALL: &str = "GET /@pnpm.e2e/pkg-with-1-dep@100.1.0 200";
const DEP_TARBALL: &str = "GET /@pnpm.e2e/dep-of-pkg-with-1-dep@100.1.0 200";
/// Install 1 also downloads both tarballs, which the store keeps for installs
/// 2 and 3. The table leaves tarballs out: in pnpm v11's test they bypass the
@@ -138,6 +137,10 @@ fn remove_if_present(result: io::Result<()>) {
fn is_tarball(request: &RecordedRequest) -> bool {
request.path.ends_with(".tgz")
|| request.path
.strip_prefix('/')
.and_then(|path| path.strip_prefix('@').or(Some(path)))
.is_some_and(|path| path.contains('@'))
}
fn document(request: &RecordedRequest) -> &'static str {
@@ -867,6 +867,18 @@ fn same_lockfiles_modules_and_tree_hoisted_and_filtered() {
}
}
fn is_tarball_request_path(path: &str) -> bool {
path.ends_with(".tgz")
|| path
.strip_prefix('/')
.and_then(|rest| {
rest.strip_prefix('@')
.unwrap_or(rest)
.split_once('@')
})
.is_some()
}
fn format_recorded_request(req: RecordedRequest) -> String {
let RecordedRequest {
method,
@@ -878,7 +890,7 @@ fn format_recorded_request(req: RecordedRequest) -> String {
..
} = req;
let path = path.strip_prefix("/npm").unwrap_or(&path);
if path.ends_with(".tgz") {
if is_tarball_request_path(path) {
return format!("{method} {path} {status}");
}
let is_abbreviated = accept
@@ -955,7 +967,11 @@ fn same_registry_request_logs_across_install_modes() {
);
let warm_reresolve_tarballs: Vec<&String> = pacquet_logs[3]
.iter()
.filter(|line| line.contains(".tgz"))
.filter(|line| {
line.split_whitespace()
.nth(1)
.is_some_and(is_tarball_request_path)
})
.collect();
assert_eq!(
warm_reresolve_tarballs,
@@ -178,11 +178,14 @@ fn switching_to_a_tar_registry_needs_update_checksums() {
npmrc_info,
..
} = env;
let initial_workspace_yaml = fs::read_to_string(workspace.join("pnpm-workspace.yaml")).unwrap();
use_tar_registry(&workspace, npmrc_info.mock_instance.url(), Some("pnpr"));
pacquet
.with_arg("install")
.assert()
.success();
fs::write(workspace.join("pnpm-workspace.yaml"), initial_workspace_yaml).unwrap();
use_tar_registry(&workspace, &tar_registry, Some("pnpr"));
fs::remove_dir_all(workspace.join("node_modules")).unwrap();
fs::remove_dir_all(&npmrc_info.store_dir).unwrap();
+6 -1
View File
@@ -860,7 +860,12 @@ fn lock_unserved_version_of_dep(workspace: &Path) {
.expect("parse pnpm-lock.yaml");
for section in ["packages", "snapshots"] {
let entries = lockfile[section].as_object_mut().expect("the lockfile has the section");
let entry = entries.remove(&locked_key).expect("the section has the locked entry");
let mut entry = entries.remove(&locked_key).expect("the section has the locked entry");
if let Some(resolution) =
entry.get_mut("resolution").and_then(serde_json::Value::as_object_mut)
{
resolution.remove("tarball");
}
entries.insert(unserved_key.clone(), entry);
}
for snapshot in lockfile["snapshots"]
@@ -65,7 +65,8 @@ async fn resolves_a_pinned_config_dep_newer_than_the_release_age_cutoff() {
dbg!(resolution);
assert!(matches!(
resolution,
LockfileResolution::Registry(registry) if registry.integrity.to_string() == integrity,
LockfileResolution::Tarball(tarball)
if tarball.integrity.as_ref().map(ToString::to_string).as_deref() == Some(integrity.as_str()),
));
assert!(
root.path()
@@ -32,6 +32,13 @@ fn project(registry: &TestRegistry) -> Project {
config.install_state_dir = modules_dir.join(".pacquet");
config.modules_dir = modules_dir;
config.registry = registry.url().to_string();
config.registry_options_by_url.insert(
registry.url().to_string(),
pnpm_lockfile::RegistryOptions {
server_type: Some(pnpm_lockfile::RegistryServerType::Pnpr),
..Default::default()
},
);
config.enable_modules_dir = false;
Project { dir, config: config.leak(), manifest }
}
@@ -59,10 +59,15 @@ async fn unknown_route_keeps_its_upstream_tarball_url() {
let registry = TestRegistry::start();
let (pnpr_url, pnpr_auth, _storage) = start_pnpr(registry.url()).await;
let outcome = PnprClient::new(pnpr_url)
.resolve(options(registry.url(), &pnpr_auth, deps([("@foo/no-deps", "1.0.0")])))
.await
.expect("install should succeed");
let mut opts = options(registry.url(), &pnpr_auth, deps([("@foo/no-deps", "1.0.0")]));
opts.routing.registries.insert(
registry.url().to_string(),
pnpm_config::RegistryDeclaration {
server_type: Some(pnpm_lockfile::RegistryServerType::Pnpr),
..pnpm_config::RegistryDeclaration::default()
},
);
let outcome = PnprClient::new(pnpr_url).resolve(opts).await.expect("install should succeed");
let lockfile = serde_json::to_value(&outcome.lockfile).expect("lockfile serializes");
let resolution = &lockfile["packages"]["@foo/no-deps@1.0.0"]["resolution"];
+3 -3
View File
@@ -77,7 +77,7 @@ Registries do not all lay out tarball URLs the way the npm registry does. pnpm o
| *undeclared* | Strict: a tarball URL is omitted only when it is exactly the canonical npm-layout URL (`<registry>/<name>/-/<scopeless-name>-<version>.tgz`). This is how every registry but `registry.npmjs.org` is read by default. |
| `npm` | The server behaves like `registry.npmjs.org`, which serves a scoped package from the percent-encoded path (`/@acme%2Fwidget/...`) as well as the unencoded one. A faithful mirror or caching proxy of the public registry is this. Built in for `registry.npmjs.org` itself. |
| `artifactory` | The server repeats the scope in a scoped package's tarball filename: `@acme/widget/-/@acme/widget-1.0.0.tgz`, where the npm layout strips it (`@acme/widget/-/widget-1.0.0.tgz`). |
| `pnpr` | The server serves a version's tarball at `<registry>/<name>@<version>`, such as `@acme/widget@1.0.0`. pnpr uses this layout on a [tar registry](#tar-registries), where the metadata carries no tarball URL, except for a version with [registry revisions](./registry-revisions.md). A pnpr `.tgz` registry advertises the plain npm layout instead, which the undeclared default already handles. |
| `pnpr` | The server serves a version's tarball at `<registry>/<name>@<version>`, such as `@acme/widget@1.0.0`. pnpr uses this layout on both `.tgz` registries and [tar registries](#tar-registries), except for a version with [registry revisions](./registry-revisions.md). |
For a JFrog Artifactory registry this is the difference between a lockfile that records a host-specific tarball URL for **every scoped package** and one that records none of them:
@@ -95,7 +95,7 @@ Values for common servers:
| JFrog Artifactory | `artifactory` |
| GitLab package registry | `artifactory`. It uses the same layout, repeating the scope in the filename. |
| A faithful mirror or caching proxy of the npm registry | `npm` |
| pnpr | `pnpr` for a [tar registry](#tar-registries). Nothing for a `.tgz` registry, which advertises the npm layout. |
| pnpr | `pnpr` |
| Verdaccio, Sonatype Nexus, Azure Artifacts | None needed. They serve the exact canonical npm layout, which the strict default already reconstructs. |
| GitHub Packages | None. Its download URLs contain a content digest that cannot be derived from the package's identity, so they are kept in the lockfile. |
| GitHub Enterprise Server | None. It serves a scoped package only from the percent-encoded path, which the strict default keeps in the lockfile and requests verbatim. |
@@ -123,7 +123,7 @@ registries:
serverType: pnpr
```
The server decides which base serves tars. In pnpr's example they are at `/npm/~zstd/`, next to the usual `.tgz` base, but a registry can serve tars from its main base too; see [tar registries in pnpr](/pnpr/configuration#tar-registries). Put `serverType: pnpr` on the tar base only. A `.tgz` base from pnpr advertises the plain npm layout, so leave `serverType` out there.
The server decides which base serves tars. In pnpr's example they are at `/npm/~zstd/`, next to the usual `.tgz` base, but a registry can serve tars from its main base too; see [tar registries in pnpr](/pnpr/configuration#tar-registries). Declare `serverType: pnpr` on any pnpr registry base so pnpm omits canonical `<name>@<version>` tarball URLs from `pnpm-lock.yaml`.
pnpm tells a tar from a `.tgz` by the first bytes of the decoded response, so nothing else needs to be declared. The lockfile keeps its `integrity` field, which for packages from a tar registry holds the hash of the tar.
@@ -21,7 +21,7 @@ mod unlocked_deps;
mod server_config;
use server_config::{
PnprServer, PnprServerPaths, RevisionMockRegistry, append_pnpr_auth_to_npmrc,
cold_mock_config_yaml, distinct_public_route_registries, mint_pnpr_token, seed_pnpr_auth,
cold_revision_mock_command, distinct_public_route_registries, mint_pnpr_token, seed_pnpr_auth,
serves_npm_under_base_path, wait_for_pnpr_ready, write_pnpr_benchmark_config,
};
@@ -7,7 +7,7 @@ use std::{
io::Write,
net::{TcpListener, TcpStream},
path::{Path, PathBuf},
process::Child,
process::{Child, Command},
thread,
time::Duration,
};
@@ -42,6 +42,10 @@ pub(super) struct PnprServer {
/// The latency proxy fronting this server, when `--pnpr-latency-ms`
/// is set. Dropped (stopping the proxy) alongside the server.
pub(super) latency_proxy: Option<LatencyProxy>,
/// Warm origin server backing a cold revision mock, spawned from the same
/// revision's `pnpr` binary so each revision speaks its own packument and
/// tarball URL format to its upstream.
pub(super) origin: Option<Box<PnprServer>>,
}
impl Drop for PnprServer {
fn drop(&mut self) {
@@ -55,6 +59,7 @@ impl Drop for PnprServer {
let _ = self.process.kill();
let _ = self.process.wait();
eprintln!("info: Terminated pnpr server pid {pid}");
self.origin = None;
}
}
/// Poll the pnpr server's TCP port until it accepts a connection. Stays
@@ -101,6 +106,36 @@ pub(super) fn cold_mock_config_yaml(storage: &Path, origin: &str) -> String {
};
serde_saphyr::to_string(&config).expect("serialize cold mock config")
}
/// Use isolated storage, wiped between iterations, with the warm mock as origin.
pub(super) fn cold_revision_mock_command(
revision: &str,
binary: &Path,
bench_dir: &Path,
mock_port: u16,
public_url: &str,
origin_url: &str,
) -> Command {
let cold_storage = bench_dir.join("cold-mock-storage");
let config_path = bench_dir.join("cold-mock-config.yaml");
fs::write(&config_path, cold_mock_config_yaml(&cold_storage, origin_url))
.expect("write cold mock config");
eprintln!(
"Serving {revision}'s tarballs from a COLD mock built from pnpr@{revision} on 127.0.0.1:{mock_port} (origin {origin_url})...",
);
let mut command = Command::new(binary);
command
.arg("--config")
.arg(&config_path)
.arg("--storage")
.arg(&cold_storage)
.arg("--listen")
.arg(format!("127.0.0.1:{mock_port}"))
.arg("--public-url")
.arg(public_url)
.arg("--packument-ttl-secs")
.arg("31536000");
command
}
/// The cold mock config, serialized rather than string-formatted so the
/// `storage` path and `origin` URL are escaped and the structure can't drift.
#[derive(Serialize)]
@@ -1,6 +1,6 @@
use super::{
BenchId, PNPR_SERVER_REGISTRY_ENV, PNPR_TARBALL_REWRITE_FROM_ENV, PnprServer, PnprServerPaths,
RevisionMockRegistry, WorkEnv, append_pnpr_auth_to_npmrc, cold_mock_config_yaml,
RevisionMockRegistry, WorkEnv, append_pnpr_auth_to_npmrc, cold_revision_mock_command,
distinct_public_route_registries, mint_pnpr_token, seed_pnpr_auth, serves_npm_under_base_path,
wait_for_pnpr_ready, write_pnpr_benchmark_config,
};
@@ -72,20 +72,8 @@ impl WorkEnv {
.expect("create revision mock stderr log");
let cold = self.options.selection.scenario.is_some_and(BenchmarkScenario::cold_pnpr_cache);
let mut command = if cold {
self.cold_revision_mock_command(
revision,
&binary,
&bench_dir,
mock_port,
&registry.origin,
)
} else {
eprintln!(
"Serving {revision}'s tarballs from a mock built from pnpr@{revision} on 127.0.0.1:{mock_port}...",
);
pnpm_registry_mock::pnpr_command_with_binary(&binary, mock_port, Some(&registry.origin))
};
let (mut command, origin) =
self.revision_mock_command(revision, &binary, &bench_dir, mock_port, &registry.origin);
self.apply_serve_timing(&mut command);
let process = command
.stdin(Stdio::null())
@@ -94,43 +82,84 @@ impl WorkEnv {
.spawn()
.expect(if cold { "spawn cold revision mock" } else { "spawn revision mock" });
let mut server = PnprServer { process, latency_proxy: None };
let mut server = PnprServer { process, latency_proxy: None, origin };
wait_for_pnpr_ready(mock_port);
self.follow_revision_npm_base_path(revision, &registry.origin, mock_port);
server.latency_proxy =
Some(self.front_revision_mock(revision, registry.listener, mock_port));
server
}
/// Use isolated storage, wiped between iterations, with the warm mock as origin.
fn cold_revision_mock_command(
fn revision_mock_command(
&self,
revision: &str,
binary: &Path,
bench_dir: &Path,
mock_port: u16,
public_url: &str,
) -> Command {
let cold_storage = bench_dir.join("cold-mock-storage");
let config_path = bench_dir.join("cold-mock-config.yaml");
fs::write(&config_path, cold_mock_config_yaml(&cold_storage, &self.registry.client))
.expect("write cold mock config");
eprintln!(
"Serving {revision}'s tarballs from a COLD mock built from pnpr@{revision} on 127.0.0.1:{mock_port} (origin {})...",
self.registry.client,
) -> (Command, Option<Box<PnprServer>>) {
if self.options.selection.scenario.is_some_and(BenchmarkScenario::cold_pnpr_cache) {
let (origin_url, origin) = self.start_cold_revision_mock_origin(binary);
let cmd = cold_revision_mock_command(
revision,
binary,
bench_dir,
mock_port,
public_url,
&origin_url,
);
(cmd, Some(Box::new(origin)))
} else {
eprintln!(
"Serving {revision}'s tarballs from a mock built from pnpr@{revision} on 127.0.0.1:{mock_port}...",
);
(
pnpm_registry_mock::pnpr_command_with_binary(binary, mock_port, Some(public_url)),
None,
)
}
}
/// Spawn a warm origin from the same revision's `pnpr` binary over the
/// shared runtime storage, fronted by the client↔registry latency and
/// bandwidth profile so the cold mock fetches packuments and tarballs from
/// an origin that speaks its own revision's URL format.
fn start_cold_revision_mock_origin(&self, binary: &Path) -> (String, PnprServer) {
let origin_port = pick_unused_port().expect("pick a port for the cold mock warm origin");
let rate_limit = mbps_to_bytes_per_sec(self.options.network.registry_bandwidth_mbps);
let latency_proxy = (self.options.network.registry_latency_ms > 0 || rate_limit.is_some())
.then(|| {
let upstream = SocketAddr::from((Ipv4Addr::LOCALHOST, origin_port));
let profile = LinkProfile {
one_way: Duration::from_millis(self.options.network.registry_latency_ms) / 2,
rate_limit,
slow_start: self.options.network.registry_slow_start,
};
LatencyProxy::spawn(upstream, profile)
.expect("spawn cold mock warm origin latency proxy")
});
let public_origin = latency_proxy
.as_ref()
.map(|proxy| format!("http://{}", proxy.addr));
let mut command = pnpm_registry_mock::pnpr_command_with_binary(
binary,
origin_port,
public_origin.as_deref(),
);
let mut command = Command::new(binary);
command
.arg("--config")
.arg(&config_path)
.arg("--storage")
.arg(&cold_storage)
.arg("--listen")
.arg(format!("127.0.0.1:{mock_port}"))
.arg("--public-url")
.arg(public_url)
.arg("--packument-ttl-secs")
.arg("31536000");
command
self.apply_serve_timing(&mut command);
let process = command
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("spawn cold mock warm origin");
let origin = PnprServer { process, latency_proxy, origin: None };
wait_for_pnpr_ready(origin_port);
let base = public_origin.unwrap_or_else(|| format!("http://127.0.0.1:{origin_port}"));
let origin_url = if serves_npm_under_base_path(origin_port) {
pnpm_registry_mock::npm_registry_url(&base)
} else {
format!("{base}/")
};
(origin_url, origin)
}
/// Start a pnpr resolver server for every `pnpr@<rev>`
/// target and write the `.pnpr-env` its `install.bash` sources. Each
@@ -206,6 +235,7 @@ impl WorkEnv {
port,
}),
latency_proxy: None,
origin: None,
};
wait_for_pnpr_ready(port);
+8 -1
View File
@@ -22,7 +22,9 @@ use pnpr_osv::{OsvIndex, format_advisory_ids};
use pnpr_package_name::CanonicalPackageName;
use pnpr_policy::Identity;
use pnpr_route::{RouteClass, RouteContext, sanitize_registry_tarball_url, strip_url_credentials};
use pnpr_upstream::tarball_basename;
use pnpr_upstream::{
tarball_basename, tarball_basename_for_package, version_route_tarball_version,
};
/// NDJSON content type for the `/-/pnpr/v0/resolve` response. One JSON object
/// per line; the client parses frames as they arrive. Excluded from the
@@ -384,6 +386,11 @@ fn vulnerability_ids_for_entry(
/// for non-standard naming so a legitimate custom registry isn't
/// misjudged. Never parses the URL strictly — the lockfile is untrusted.
pub(super) fn tarball_url_version<'a>(url: &'a str, name: &str) -> Option<&'a str> {
if let Ok(canonical) = CanonicalPackageName::parse(name, pnpr_package_name::Ecosystem::Npm)
&& let Some(version) = version_route_tarball_version(url, &canonical)
{
return Some(version);
}
let last = url.rsplit('/').next()?;
let last = last
.split(['?', '#'])
@@ -2,7 +2,8 @@ use super::{
Arc, CanonicalPackageName, HashMap, Identity, Lockfile, LockfileResolution, PacquetConfig,
RouteClass, RouteContext, TarballResolution, is_git_hosted_tarball_url, is_http_tarball_url,
pick_registry_for_package, sanitize_registry_tarball_url, strip_url_credentials,
tarball_basename, tarball_url_and_integrity,
tarball_basename, tarball_basename_for_package, tarball_url_and_integrity,
version_route_tarball_version,
};
#[derive(Clone)]
@@ -47,16 +48,15 @@ impl TarballRouter {
// token. A genuinely public tarball is anonymously fetchable, so the
// sanitized URL still works.
RouteClass::Public => sanitize_registry_tarball_url(tarball_url),
RouteClass::Hosted { .. } => pnpr_tarball_url(
&self.public_url,
package,
&tarball_filename(package, version, tarball_url),
),
RouteClass::Hosted { .. } => {
pnpr_tarball_url(&self.public_url, package, version, tarball_url)
}
RouteClass::Proxied { alias, .. } => upstream_endpoint_tarball_url(
&self.public_url,
&alias,
package,
&tarball_filename(package, version, tarball_url),
version,
tarball_url,
),
}
}
@@ -129,16 +129,15 @@ impl TarballRouter {
// malicious/compromised registry embedded in `dist.tarball` is
// stripped first, so pnpr never streams or caches it.
RouteClass::Public => strip_url_credentials(tarball_url),
RouteClass::Hosted { .. } => pnpr_tarball_url(
&self.public_url,
package,
&tarball_filename(package, version, tarball_url),
),
RouteClass::Hosted { .. } => {
pnpr_tarball_url(&self.public_url, package, version, tarball_url)
}
RouteClass::Proxied { alias, .. } => upstream_endpoint_tarball_url(
&self.public_url,
&alias,
package,
&tarball_filename(package, version, tarball_url),
version,
tarball_url,
),
}
}
@@ -157,18 +156,28 @@ impl TarballRouter {
}
}
pub(super) fn tarball_filename(package: &str, version: &str, tarball_url: &str) -> String {
if let Some(basename) = tarball_basename(tarball_url) {
return basename.to_string();
}
match CanonicalPackageName::parse(package, pnpr_package_name::Ecosystem::Npm) {
Ok(name) => name.tarball_name_for_version(version),
Err(_) => format!("{package}-{version}.tgz"),
pub(super) fn tarball_route_tail(package: &str, version: &str, tarball_url: &str) -> String {
if let Ok(name) = CanonicalPackageName::parse(package, pnpr_package_name::Ecosystem::Npm)
&& let Some(route_version) = version_route_tarball_version(tarball_url, &name)
{
return format!("{package}@{route_version}");
}
let filename = match CanonicalPackageName::parse(package, pnpr_package_name::Ecosystem::Npm) {
Ok(name) => tarball_basename_for_package(tarball_url, &name)
.map_or_else(|| name.tarball_name_for_version(version), std::borrow::Cow::into_owned),
Err(_) => tarball_basename(tarball_url)
.map_or_else(|| format!("{package}-{version}.tgz"), str::to_string),
};
format!("{package}/-/{filename}")
}
pub(super) fn pnpr_tarball_url(public_url: &str, package: &str, filename: &str) -> String {
format!("{}/{package}/-/{filename}", npm_endpoint(public_url))
pub(super) fn pnpr_tarball_url(
public_url: &str,
package: &str,
version: &str,
tarball_url: &str,
) -> String {
format!("{}/{}", npm_endpoint(public_url), tarball_route_tail(package, version, tarball_url))
}
/// The registry-endpoint URL a proxied route's tarball is served through.
@@ -176,9 +185,14 @@ pub(super) fn upstream_endpoint_tarball_url(
public_url: &str,
upstream: &str,
package: &str,
filename: &str,
version: &str,
tarball_url: &str,
) -> String {
format!("{}/~{upstream}/{package}/-/{filename}", npm_endpoint(public_url))
format!(
"{}/~{upstream}/{}",
npm_endpoint(public_url),
tarball_route_tail(package, version, tarball_url),
)
}
fn npm_endpoint(public_url: &str) -> String {
+1 -1
View File
@@ -187,7 +187,7 @@ use pnpr_upstream::{
CacheValidators, FetchOutcome, PackumentFetch, Upstream, abbreviate_packument,
extract_upstream_revision_integrity, extract_upstream_version_manifest,
extract_version_manifest, rewrite_tarball_urls, rewrite_upstream_tarball_urls,
tarball_basename,
tarball_basename_for_package, version_route_tarball_version,
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@@ -1,4 +1,4 @@
use pnpr_upstream::tarball_basename;
use pnpr_upstream::{tarball_basename_for_package, version_route_tarball_version};
use serde_json::Map;
use super::{CanonicalPackageName, RegistryError, Value};
@@ -184,12 +184,14 @@ pub(super) fn check_integrity_immutable(
}
/// `dist.tarball` of a published version may not change, and an update that
/// omits it has the stored one put back.
/// omits it or submits a round-tripped `<pkg>@<version>` URL has the stored
/// `.tgz` URL put back.
///
/// Basenames are compared, not URLs: the round-trip carries the rewritten URL
/// (see [`pnpr_upstream::rewrite_tarball_urls`]) while the hosted side keeps
/// the original, and [`served_tarball_basename`] applies the same
/// version-derived fallback so a basename-less stored URL is still pinned.
/// Canonical tarball identities are compared, not URLs: the round-trip carries
/// the rewritten URL (see [`pnpr_upstream::rewrite_tarball_urls`], which emits
/// `{public_url}/{pkg}@{version}`) while the hosted side keeps the `.tgz` URL,
/// and [`served_tarball_basename`] applies the same version-derived fallback so
/// a basename-less stored URL is still pinned.
pub(super) fn check_tarball_immutable(
entry: &PublishedVersion<'_>,
name: &CanonicalPackageName,
@@ -197,35 +199,53 @@ pub(super) fn check_tarball_immutable(
) -> Option<RegistryError> {
let version = entry.version;
let stored_basename = served_tarball_basename(entry.existing, name)?;
let incoming_basename = entry.manifest
let incoming_url = entry.manifest
.get("dist")
.and_then(|dist| dist.get("tarball"))
.and_then(Value::as_str)
.and_then(tarball_basename);
.and_then(Value::as_str);
let incoming_basename =
incoming_url.and_then(|url| tarball_basename_or_version_route(url, name));
match incoming_basename {
Some(submitted) if submitted != stored_basename => Some(RegistryError::BadRequest {
reason: format!("dist.tarball for the published version {version:?} is immutable"),
}),
Some(_) => None,
Some(_) => {
if incoming_url.is_some_and(|url| is_version_route_url(url, name)) {
restore_stored_tarball(entry, restore);
}
None
}
None => {
let refusal = require_object_dist(entry.manifest, version);
if refusal.is_none() {
let stored = entry.existing
.get("dist")
.and_then(|dist| dist.get("tarball"))
.cloned()
.unwrap_or(Value::Null);
restore.push((version.clone(), "tarball".to_string(), stored));
restore_stored_tarball(entry, restore);
}
refusal
}
}
}
fn restore_stored_tarball(
entry: &PublishedVersion<'_>,
restore: &mut Vec<(String, String, Value)>,
) {
let stored = entry.existing
.get("dist")
.and_then(|dist| dist.get("tarball"))
.cloned()
.unwrap_or(Value::Null);
restore.push((entry.version.clone(), "tarball".to_string(), stored));
}
fn is_version_route_url(url: &str, pkg: &CanonicalPackageName) -> bool {
version_route_tarball_version(url, pkg).is_some()
}
/// The tarball basename a version is actually served under, mirroring
/// [`pnpr_upstream::rewrite_tarball_urls`]: the `dist.tarball` URL's own basename when it has
/// one, otherwise the version-derived canonical name the rewrite falls back to.
/// Returns `None` when the manifest carries no string `dist.tarball` to serve.
/// one (normalized when it names `<pkg>@<version>`), otherwise the version-derived canonical name
/// the rewrite falls back to. Returns `None` when the manifest carries no string `dist.tarball` to
/// serve.
pub(super) fn served_tarball_basename(
manifest: &Value,
pkg: &CanonicalPackageName,
@@ -234,13 +254,18 @@ pub(super) fn served_tarball_basename(
.get("dist")
.and_then(|dist| dist.get("tarball"))
.and_then(Value::as_str)?;
if let Some(basename) = tarball_basename(url) {
return Some(basename.to_owned());
if let Some(basename) = tarball_basename_or_version_route(url, pkg) {
return Some(basename);
}
let version = manifest.get("version").and_then(Value::as_str)?;
Some(pkg.tarball_name_for_version(version))
}
fn tarball_basename_or_version_route(url: &str, pkg: &CanonicalPackageName) -> Option<String> {
let basename = tarball_basename_for_package(url, pkg)?;
Some(pkg.canonicalize_tarball_name(&basename).unwrap_or_else(|_| basename.into_owned()))
}
/// Reject a published version whose `dist` isn't an object: a restore needs an
/// object to write into, so otherwise it would no-op and persist the version
/// without the field — the stripping this guards against.
@@ -300,3 +300,46 @@ fn strips_round_tripped_dist_pnpr_overlay() {
assert!(enforce_published_version_immutability(&hosted, &pkg, &mut incoming).is_none());
assert!(incoming["versions"]["1.0.0"]["dist"].get("pnpr").is_none());
}
#[test]
fn accepts_round_tripped_version_route_tarball_url_and_restores_stored_url() {
let pkg =
CanonicalPackageName::parse("@acme/mypkg", pnpr_package_name::Ecosystem::Npm).unwrap();
let hosted = json!({
"versions": {
"1.0.0": {
"name": "@acme/mypkg",
"version": "1.0.0",
"dist": {
"integrity": "sha512-test",
"tarball": "http://localhost/npm/@acme/mypkg/-/mypkg-1.0.0.tgz",
}
}
}
});
let mut incoming = json!({
"versions": {
"1.0.0": {
"name": "@acme/mypkg",
"version": "1.0.0",
"dist": {
"integrity": "sha512-test",
"tarball": "http://example.test/npm/@acme/mypkg@1.0.0",
}
}
}
});
assert!(enforce_published_version_immutability(&hosted, &pkg, &mut incoming).is_none());
assert_eq!(
incoming["versions"]["1.0.0"]["dist"]["tarball"],
json!("http://localhost/npm/@acme/mypkg/-/mypkg-1.0.0.tgz"),
);
let mut wrong_version = incoming;
wrong_version["versions"]["1.0.0"]["dist"]["tarball"] =
json!("http://example.test/npm/@acme/mypkg@2.0.0");
let err = enforce_published_version_immutability(&hosted, &pkg, &mut wrong_version).unwrap();
assert!(matches!(err, RegistryError::BadRequest { .. }));
}
@@ -2,7 +2,8 @@ use super::{
ABBREVIATED_CONTENT_TYPE, Arc, Body, CanonicalPackageName, DistView, HeaderMap, IndexMap,
Integrity, RegistryError, Response, StatusCode, Utc, Value, abbreviate_packument,
describe_tars, filter_osv_vulnerable_versions, header, rewrite_tarball_urls,
rewrite_upstream_tarball_urls, streaming, tarball_basename,
rewrite_upstream_tarball_urls, streaming, tarball_basename_for_package,
version_route_tarball_version,
};
/// The version a tarball request resolves to, plus that version's declared
@@ -12,6 +13,7 @@ use super::{
/// resolves to the right version, integrity, and OSV identity.
pub(super) struct TarballDist {
pub(super) version: String,
pub(super) version_route: Option<String>,
pub(super) integrity: Integrity,
pub(super) unpacked_size: Option<u64>,
}
@@ -58,7 +60,7 @@ pub(super) fn expected_tarball_dist(
let dist = manifest.dist.as_ref()?;
dist.tarball
.as_deref()
.and_then(tarball_basename)
.and_then(|url| tarball_basename_for_package(url, name))
.is_some_and(|basename| basename == filename)
.then_some((version, dist))
});
@@ -77,7 +79,16 @@ pub(super) fn expected_tarball_dist(
));
}
let integrity = declared_tarball_integrity(dist, name, filename, version)?;
Ok(Some(TarballDist { version: version.clone(), integrity, unpacked_size: dist.unpacked_size }))
let version_route = dist.tarball
.as_deref()
.and_then(|url| version_route_tarball_version(url, name))
.map(str::to_string);
Ok(Some(TarballDist {
version: version.clone(),
version_route,
integrity,
unpacked_size: dist.unpacked_size,
}))
}
/// Prefer SRI and fall back to legacy SHA-1, while refusing unverified bytes.
@@ -343,17 +343,30 @@ pub(super) async fn put_scoped_packument_revision(
}
/// `DELETE {base}/{pkg}/-rev/{rev}` — remove a whole package
/// (`pnpm unpublish --force`).
/// (`pnpm unpublish --force`), or remove one version's tarball when the segment
/// is `{pkg}@{version}` (`DELETE {dist.tarball}/-rev/{rev}`).
pub(super) async fn unpublish_package(
State(state): State<AppState>,
AuthedCaller(identity): AuthedCaller,
TargetRegistry(registry): TargetRegistry,
Path(path): Path<NamePath>,
) -> Response {
if let Some((pkg_name, version)) = split_package_identifier(&path.name) {
return unpublish_version_tarball(
&state,
&identity,
registry.as_deref(),
pkg_name,
version,
)
.await;
}
delete_package(&state, &identity, registry.as_deref(), &path.name).await
}
/// `DELETE {base}/@{scope}/{pkg}/-rev/{rev}` — remove a whole scoped package.
/// `DELETE {base}/@{scope}/{pkg}/-rev/{rev}` — remove a whole scoped package,
/// or remove one scoped version's tarball when the second segment is
/// `{pkg}@{version}` (`DELETE {dist.tarball}/-rev/{rev}`).
pub(super) async fn unpublish_scoped_package(
State(state): State<AppState>,
AuthedCaller(identity): AuthedCaller,
@@ -364,10 +377,33 @@ pub(super) async fn unpublish_scoped_package(
if !scope.starts_with('@') {
return not_found();
}
if let Some((bare_name, version)) = split_package_identifier(&name) {
let full = format!("{scope}/{bare_name}");
return unpublish_version_tarball(&state, &identity, registry.as_deref(), &full, version)
.await;
}
let full = format!("{scope}/{name}");
delete_package(&state, &identity, registry.as_deref(), &full).await
}
async fn unpublish_version_tarball(
state: &AppState,
identity: &Identity,
registry: Option<&str>,
raw_name: &str,
version: &str,
) -> Response {
if super::super::revision_tarballs::parse_revision_version(version).is_some() {
return not_found();
}
let canonical = match CanonicalPackageName::parse(raw_name, Ecosystem::Npm) {
Ok(name) => name,
Err(err) => return err.into_response(),
};
let filename = canonical.tarball_name_for_version(version);
delete_tarball(state, identity, registry, raw_name, &filename).await
}
/// `DELETE {base}/{pkg}/-/{filename}/-rev/{rev}` — remove one version's
/// tarball, a step of `pnpm unpublish <pkg>@<version>`.
pub(super) async fn unpublish_tarball(
@@ -10,6 +10,7 @@ use std::collections::HashSet;
struct MissingIntegrityTarball {
version: String,
filename: String,
version_route: Option<String>,
needs_integrity: bool,
expected_shasum: Option<Integrity>,
}
@@ -190,7 +191,11 @@ impl PackumentIntegrityResolver<'_> {
let fetched = timed(
"tarball:integrity_fetch",
self.name.as_str(),
self.upstream.fetch_tarball_response(self.name, &candidate.filename),
self.upstream.fetch_tarball_or_version_response(
self.name,
&candidate.filename,
candidate.version_route.as_deref(),
),
)
.await?;
let response = match fetched {
@@ -326,6 +331,8 @@ fn packument_tarball(
return Ok(None);
};
let filename = packument_tarball_filename(tarball, name, version)?;
let version_route =
pnpr_upstream::version_route_tarball_version(tarball, name).map(str::to_string);
let needs_integrity = needs_integrity(dist, name, &filename, version)?;
let expected_shasum = if needs_integrity {
legacy_shasum_integrity(dist, name, &filename, version)?
@@ -335,6 +342,7 @@ fn packument_tarball(
Ok(Some(MissingIntegrityTarball {
version: version.to_string(),
filename,
version_route,
needs_integrity,
expected_shasum,
}))
@@ -345,9 +353,9 @@ fn packument_tarball_filename(
name: &CanonicalPackageName,
version: &str,
) -> Result<String, RegistryError> {
let basename = pnpr_upstream::tarball_basename(tarball);
let basename = pnpr_upstream::tarball_basename_for_package(tarball, name);
let fallback = name.tarball_name_for_version(version);
let raw_filename = basename.unwrap_or(&fallback);
let raw_filename = basename.as_deref().unwrap_or(&fallback);
let (filename, _) = tarball_cache_name(name, raw_filename)
.map_err(|err| {
tarball_integrity_error(
@@ -13,6 +13,7 @@ fn pinned(version: &str, shasum_of: Option<&[u8]>) -> MissingIntegrityTarball {
MissingIntegrityTarball {
version: version.to_string(),
filename: format!("foo-{version}.tgz"),
version_route: None,
needs_integrity: true,
expected_shasum: shasum_of.map(|bytes| digest(Algorithm::Sha1, bytes)),
}
@@ -89,7 +89,12 @@ pub(super) async fn serve_authorized_upstream_tarball(
return response;
}
let dist = bind_tarball_to_packument(state, upstream, namespace, name, filename, ttl).await;
let TarballDist { version, integrity, unpacked_size } = match dist {
let TarballDist {
version,
version_route,
integrity,
unpacked_size,
} = match dist {
Ok(Some(dist)) => dist,
Ok(None) => return not_found(),
Err(err) => return err.into_response(),
@@ -97,8 +102,14 @@ pub(super) async fn serve_authorized_upstream_tarball(
if let Err(err) = recheck_osv(state, name, &version, parsed_version) {
return err.into_response();
}
let tarball =
UpstreamTarball { namespace, name, filename, integrity: &integrity, unpacked_size };
let tarball = UpstreamTarball {
namespace,
name,
filename,
version_route: version_route.as_deref(),
integrity: &integrity,
unpacked_size,
};
if upstream.caches()
&& state.inner.osv_index.is_some()
&& let Some(response) = cached_upstream_tarball(state, namespace, name, filename).await
@@ -220,6 +231,7 @@ pub(super) struct UpstreamTarball<'a> {
pub(super) namespace: &'a str,
pub(super) name: &'a CanonicalPackageName,
pub(super) filename: &'a str,
pub(super) version_route: Option<&'a str>,
pub(super) integrity: &'a Integrity,
pub(super) unpacked_size: Option<u64>,
}
@@ -235,13 +247,14 @@ pub(super) async fn fetch_upstream_tarball(
namespace,
name,
filename,
version_route,
integrity,
unpacked_size: _,
} = tarball;
let fetched = timed(
"tarball:upstream_fetch",
name.as_str(),
upstream.fetch_tarball_response(name, filename),
upstream.fetch_tarball_or_version_response(name, filename, version_route),
)
.await;
let response = match fetched {
+30 -15
View File
@@ -20,8 +20,8 @@ use super::{
declared_tarball_integrity, ensure_osv_allowed, is_tar_registry, not_found,
read_source_packument, recover_stale_upstream_packument, resolve_registry_source,
serve_bound_upstream_tarball, serve_registry_tarball, serve_revision_tarball, tar_decode_limit,
tar_response, tarball_basename, tarball_cache_name, tgz_integrity_key, timed,
upstream_cache_namespace,
tar_response, tarball_basename_for_package, tarball_cache_name, tgz_integrity_key, timed,
upstream_cache_namespace, version_route_tarball_version,
};
use axum::response::IntoResponse;
use pnpr_storage::DocumentStamp;
@@ -125,19 +125,19 @@ impl VersionRequest<'_> {
tar_response(self.state, headers, response, source).await
}
/// Serve the version's `.tgz` under its canonical filename without its
/// packument: a hosted blob, which publish names canonically, or a cached
/// upstream `.tgz`, as the filename route serves one. `None` leaves the
/// version to its packument, including a `dist.tarball` that names
/// another file.
/// Serve the version's `.tgz` without reading its packument: a hosted
/// blob, which publish names canonically, or a cached upstream `.tgz`
/// whose filename [`VersionTgzCache`] already bound to this version.
/// `None` leaves the version to its packument, including a `dist.tarball`
/// that names another file.
async fn serve_canonical_tgz(
&self,
name: &CanonicalPackageName,
source: &RegistrySource,
) -> Option<Response> {
let canonical = name.tarball_name_for_version(self.version);
match source {
RegistrySource::Hosted(_) => {
let canonical = name.tarball_name_for_version(self.version);
let response = serve_registry_tarball(
self.state,
self.identity,
@@ -155,7 +155,13 @@ impl VersionRequest<'_> {
Ok(cached) => cached?,
Err(err) => return Some(err.into_response()),
};
cached_upstream_tarball(self.state, &cached.namespace, name, &canonical).await
let tgz = self.state.inner.proxy.version_tgzs.get_bound(
&cached.namespace,
name,
self.version,
)?;
let (filename, _) = tarball_cache_name(name, &tgz.filename).ok()?;
cached_upstream_tarball(self.state, &cached.namespace, name, &filename).await
}
_ => None,
}
@@ -213,9 +219,15 @@ impl VersionRequest<'_> {
else {
return Ok(None);
};
// The stamp was taken before the read, so a packument rewritten in
// between only costs the next request a miss.
Ok(version_tgz(&packument, self.version).map(|tgz| (tgz, stamp.map(DistOrigin::Packument))))
let stamp = match (stamp, cached) {
(Some(stamp), _) => Some(stamp),
(None, Some(cached)) => {
self.state.inner.storage.upstream_document_stamp(&cached.namespace, name).await?
}
(None, None) => None,
};
Ok(version_tgz(&packument, name, self.version)
.map(|tgz| (tgz, stamp.map(DistOrigin::Packument))))
}
/// The version's dist from its manifest, or from the stale cached
@@ -241,7 +253,7 @@ impl VersionRequest<'_> {
recover_stale_upstream_packument(self.state, &cached?.namespace, upstream, name, err)
.await
.ok()??;
version_tgz(&stale, self.version).map(|tgz| (tgz, None))
version_tgz(&stale, name, self.version).map(|tgz| (tgz, None))
}
/// The version's dist from `<upstream>/<name>/<version>`.
@@ -258,8 +270,10 @@ impl VersionRequest<'_> {
)
.await;
match fetched {
Ok(FetchOutcome::Ok(manifest)) => version_manifest_tgz(&manifest.bytes, self.version)
.map_or(ManifestRead::Unusable, ManifestRead::Dist),
Ok(FetchOutcome::Ok(manifest)) => {
version_manifest_tgz(&manifest.bytes, name, self.version)
.map_or(ManifestRead::Unusable, ManifestRead::Dist)
}
Ok(FetchOutcome::NotFound) => ManifestRead::Unusable,
Err(
err @ (RegistryError::Upstream { .. } | RegistryError::UpstreamUnavailable { .. }),
@@ -366,6 +380,7 @@ impl VersionRequest<'_> {
namespace: &namespace,
name,
filename: &filename,
version_route: tgz.version_route.as_deref(),
integrity: &integrity,
unpacked_size: tgz.dist.unpacked_size,
};
@@ -1,5 +1,6 @@
use super::{
CanonicalPackageName, DistBlock, PackumentDists, Value, tarball_basename, tgz_integrity_key,
CanonicalPackageName, DistBlock, PackumentDists, Value, tarball_basename_for_package,
tgz_integrity_key, version_route_tarball_version,
};
use pnpr_storage::DocumentStamp;
use std::{
@@ -10,13 +11,24 @@ use std::{
pub(super) struct VersionTgz {
pub(super) filename: String,
pub(super) version_route: Option<String>,
pub(super) dist: DistBlock,
pub(super) integrity_key: Option<String>,
}
pub(super) fn version_tgz(packument: &[u8], version: &str) -> Option<VersionTgz> {
pub(super) fn version_tgz(
packument: &[u8],
name: &CanonicalPackageName,
version: &str,
) -> Option<VersionTgz> {
let mut packument: PackumentDists = serde_json::from_slice(packument).ok()?;
dist_tgz(packument.versions.shift_remove(version)?.dist?)
let tgz = dist_tgz(name, packument.versions.shift_remove(version)?.dist?)?;
let shared_filename = packument.versions
.values()
.filter_map(|manifest| manifest.dist.as_ref()?.tarball.as_deref())
.filter_map(|url| tarball_basename_for_package(url, name))
.any(|basename| basename == tgz.filename);
(!shared_filename).then_some(tgz)
}
#[derive(serde::Deserialize)]
@@ -28,20 +40,25 @@ struct VersionManifest {
/// The dist of a version manifest for `version`. An upstream resolves a
/// dist-tag at the same URL, so a manifest of another version is refused. A
/// dist without a valid integrity is left to the packument, which carries the
/// integrity pnpr pins for it.
pub(super) fn version_manifest_tgz(manifest: &[u8], version: &str) -> Option<VersionTgz> {
/// dist without a valid integrity or with a non-canonical filename is left to
/// the packument, which carries the integrity pnpr pins for it and checks that
/// no other version claims the same filename.
pub(super) fn version_manifest_tgz(
manifest: &[u8],
name: &CanonicalPackageName,
version: &str,
) -> Option<VersionTgz> {
let manifest: VersionManifest = serde_json::from_slice(manifest).ok()?;
let dist = manifest.dist.filter(|dist| dist.integrity.is_some())?;
let tgz = (manifest.version == version).then(|| dist_tgz(dist))??;
tgz.integrity_key.is_some().then_some(tgz)
let tgz = (manifest.version == version).then(|| dist_tgz(name, dist))??;
(tgz.integrity_key.is_some() && tgz.filename == name.tarball_name_for_version(version))
.then_some(tgz)
}
fn dist_tgz(dist: DistBlock) -> Option<VersionTgz> {
let filename = dist.tarball
.as_deref()
.and_then(tarball_basename)?
.to_string();
fn dist_tgz(name: &CanonicalPackageName, dist: DistBlock) -> Option<VersionTgz> {
let url = dist.tarball.as_deref()?;
let filename = tarball_basename_for_package(url, name)?.into_owned();
let version_route = version_route_tarball_version(url, name).map(str::to_string);
let mut fields = serde_json::Map::new();
if let Some(integrity) = &dist.integrity {
fields.insert("integrity".to_string(), Value::String(integrity.clone()));
@@ -49,7 +66,7 @@ fn dist_tgz(dist: DistBlock) -> Option<VersionTgz> {
if let Some(shasum) = &dist.shasum {
fields.insert("shasum".to_string(), Value::String(shasum.clone()));
}
Some(VersionTgz { filename, integrity_key: tgz_integrity_key(&fields), dist })
Some(VersionTgz { filename, version_route, integrity_key: tgz_integrity_key(&fields), dist })
}
/// Versions kept before the cache drops some.
@@ -80,6 +97,17 @@ struct CachedVersionTgz {
}
impl VersionTgzCache {
pub(super) fn get_bound(
&self,
namespace: &str,
name: &CanonicalPackageName,
version: &str,
) -> Option<Arc<VersionTgz>> {
let entries = self.entries.lock().expect("version dist cache lock poisoned");
let key = (namespace.to_string(), name.as_str().to_string(), version.to_string());
entries.get(&key).map(|cached| Arc::clone(&cached.tgz))
}
pub(super) fn get(
&self,
namespace: &str,
@@ -10,6 +10,7 @@ const TTL: Duration = Duration::from_mins(1);
fn tgz() -> Arc<VersionTgz> {
Arc::new(VersionTgz {
filename: "foo-1.0.0.tgz".to_string(),
version_route: None,
dist: serde_json::from_str("{}").unwrap(),
integrity_key: None,
})
@@ -133,7 +133,7 @@ async fn authenticated_publish_writes_manifest_and_tarball() {
let served = body_json(response.into_body()).await;
assert_eq!(
served["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/mypkg/-/mypkg-1.0.0.tgz",
"http://example.test/npm/mypkg@1.0.0",
);
}
@@ -318,6 +318,80 @@ async fn unpublish_scoped_tarball_via_six_segment_route() {
assert!(!local_blob_path(&storage, "", "@scope/unpub", "unpub-1.0.0.tgz").exists());
}
#[tokio::test]
async fn unpublish_tarball_via_version_route_url() {
let tmp = TempDir::new().unwrap();
let storage = tmp.path().to_path_buf();
let app = router(static_config(storage.clone()));
let (app, token) = add_user_and_get_token(app, "alice", "secret").await;
for (pkg, put_path, delete_path, blob_name) in [
(
"unpub-ver",
"/npm/unpub-ver",
"/npm/unpub-ver@1.0.0/-rev/anything",
"unpub-ver-1.0.0.tgz",
),
(
"@scope/unpub-ver",
"/npm/@scope/unpub-ver",
"/npm/@scope/unpub-ver@1.0.0/-rev/anything",
"unpub-ver-1.0.0.tgz",
),
] {
let body = publish_doc(pkg, "1.0.0", b"bytes");
let request = Request::put(put_path)
.header("content-type", "application/json")
.header("Authorization", format!("Bearer {token}"))
.body(Body::from(serde_json::to_vec(&body).unwrap()))
.unwrap();
let publish = app
.clone()
.oneshot(request)
.await
.unwrap();
assert_eq!(publish.status(), StatusCode::CREATED);
assert!(local_blob_path(&storage, "", pkg, blob_name).exists());
let request = Request::delete(delete_path)
.header("Authorization", format!("Bearer {token}"))
.body(Body::empty())
.unwrap();
let response = app
.clone()
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert!(!local_blob_path(&storage, "", pkg, blob_name).exists());
}
// Reserved `+r<digits>` revision identifiers must not delete a hosted `.tgz` blob.
let body = publish_doc("unpub-rev", "1.0.0+r0", b"bytes");
let request = Request::put("/npm/unpub-rev")
.header("content-type", "application/json")
.header("Authorization", format!("Bearer {token}"))
.body(Body::from(serde_json::to_vec(&body).unwrap()))
.unwrap();
assert_eq!(
app.clone()
.oneshot(request)
.await
.unwrap()
.status(),
StatusCode::CREATED,
);
assert!(local_blob_path(&storage, "", "unpub-rev", "unpub-rev-1.0.0+r0.tgz").exists());
let request = Request::delete("/npm/unpub-rev@1.0.0+r0/-rev/anything")
.header("Authorization", format!("Bearer {token}"))
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
assert!(local_blob_path(&storage, "", "unpub-rev", "unpub-rev-1.0.0+r0.tgz").exists());
}
#[tokio::test]
async fn missing_unpublish_policy_denies_destructive_writes() {
let tmp = TempDir::new().unwrap();
+1 -1
View File
@@ -192,7 +192,7 @@ async fn batch_publish_supports_scoped_packages_with_libnpmpublish_attachment_na
let served_packument = body_json(packument_response.into_body()).await;
assert_eq!(
served_packument["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/@scope/wire-form/-/wire-form-1.0.0.tgz",
"http://example.test/npm/@scope/wire-form@1.0.0",
);
let served = app
+4 -10
View File
@@ -55,11 +55,11 @@ async fn serves_scoped_packument_from_storage() {
// The on-disk packument has a Verdaccio-form tarball URL —
// `http://localhost:4873/@foo/no-deps/-/@foo/no-deps-1.0.0.tgz`.
// After our rewrite the client should see the npm-spec form
// After our rewrite the client should see the `<name>@<version>` form
// pointed at our `public_url`.
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
format!("{PUBLIC_URL}/npm/@foo/no-deps/-/no-deps-1.0.0.tgz"),
format!("{PUBLIC_URL}/npm/@foo/no-deps@1.0.0"),
);
// Other fields (integrity, shasum, name, version) should pass
// through untouched.
@@ -149,10 +149,7 @@ async fn abbreviated_accept_header_strips_packument() {
assert!(version_obj.get("contributors").is_none(), "abbreviated form should drop contributors");
assert_eq!(version_obj["name"], "@foo/no-deps");
assert_eq!(version_obj["version"], "1.0.0");
assert_eq!(
version_obj["dist"]["tarball"],
format!("{PUBLIC_URL}/npm/@foo/no-deps/-/no-deps-1.0.0.tgz"),
);
assert_eq!(version_obj["dist"]["tarball"], format!("{PUBLIC_URL}/npm/@foo/no-deps@1.0.0"));
assert!(
version_obj["dist"]["integrity"]
.as_str()
@@ -226,10 +223,7 @@ async fn serves_version_manifest_by_dist_tag() {
let manifest: Value = serde_json::from_slice(&body_bytes(response.into_body()).await).unwrap();
assert_eq!(manifest["name"], "@foo/no-deps");
assert_eq!(manifest["version"], "1.0.0");
assert_eq!(
manifest["dist"]["tarball"],
format!("{PUBLIC_URL}/npm/@foo/no-deps/-/no-deps-1.0.0.tgz"),
);
assert_eq!(manifest["dist"]["tarball"], format!("{PUBLIC_URL}/npm/@foo/no-deps@1.0.0"));
// The response is the single-version manifest, not the whole
// packument — `versions` shouldn't be there.
assert!(manifest.get("versions").is_none());
+1 -1
View File
@@ -79,7 +79,7 @@ async fn publishes_to_and_serves_from_the_object_store() {
let served = body_json(response.into_body()).await;
assert_eq!(
served["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/mypkg/-/mypkg-1.0.0.tgz",
"http://example.test/npm/mypkg@1.0.0",
);
// The tarball streams back byte-for-byte out of the bucket.
@@ -342,7 +342,7 @@ async fn registry_addressed_surface_serves_dist_tags_unpublish_whoami_search_and
let manifest = body_json(manifest.into_body()).await;
assert_eq!(
manifest["dist"]["tarball"],
json!("http://example.test/npm/~acme/@acme/widget/-/widget-1.0.0.tgz"),
json!("http://example.test/npm/~acme/@acme/widget@1.0.0"),
);
assert!(local_blob_path(tmp.path(), "acme", "@acme/widget", "widget-1.0.0.tgz").exists());
+3 -12
View File
@@ -53,10 +53,7 @@ async fn packument_is_proxied_cached_and_rewritten() {
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body: Value = serde_json::from_slice(&body_bytes(response.into_body()).await).unwrap();
assert_eq!(
body["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/foo/-/foo-1.0.0.tgz",
);
assert_eq!(body["versions"]["1.0.0"]["dist"]["tarball"], "http://example.test/npm/foo@1.0.0");
assert_eq!(body["versions"]["1.0.0"]["dist"]["shasum"], sha1_hex_of(tarball));
assert_eq!(body["versions"]["1.0.0"]["dist"]["integrity"], sha512_integrity(tarball));
@@ -541,10 +538,7 @@ async fn packument_is_gzipped_for_clients_that_accept_it() {
let mut decoded = Vec::new();
GzDecoder::new(&gzipped[..]).read_to_end(&mut decoded).expect("decode gzip");
let body: Value = serde_json::from_slice(&decoded).unwrap();
assert_eq!(
body["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/foo/-/foo-1.0.0.tgz",
);
assert_eq!(body["versions"]["1.0.0"]["dist"]["tarball"], "http://example.test/npm/foo@1.0.0");
mock.assert_async().await;
}
@@ -582,10 +576,7 @@ async fn packument_is_zstd_compressed_for_clients_that_prefer_it() {
Some("zstd"),
);
let body = decode_zstd(&body_bytes(response.into_body()).await).await;
assert_eq!(
body["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/foo/-/foo-1.0.0.tgz",
);
assert_eq!(body["versions"]["1.0.0"]["dist"]["tarball"], "http://example.test/npm/foo@1.0.0");
mock.assert_async().await;
}
@@ -682,7 +682,7 @@ async fn a_single_npm_ecosystem_still_answers_under_its_prefix() {
.as_str()
.unwrap()
.to_string();
assert_eq!(advertised, "http://example.test/npm/npm/-/npm-10.0.0.tgz");
assert_eq!(advertised, "http://example.test/npm/npm@10.0.0");
let fetched = app
.oneshot(
+96 -4
View File
@@ -189,7 +189,7 @@ async fn upstream_endpoint_serves_packument_with_endpoint_rewritten_tarballs() {
// URL is canonical for the client's configured registry (integrity-only).
assert_eq!(
body["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/~npmjs/foo/-/foo-1.0.0.tgz",
"http://example.test/npm/~npmjs/foo@1.0.0",
);
mock.assert_async().await;
}
@@ -812,7 +812,7 @@ async fn non_canonical_upstream_tarball_basename_is_served() {
let tmp = TempDir::new().unwrap();
let app = router(config_for(&upstream.url(), tmp.path().to_path_buf()));
// The served packument advertises the preserved basename on this server.
// The served packument advertises the `<name>@<version>` route on this server.
let served = app
.clone()
.oneshot(
@@ -824,9 +824,22 @@ async fn non_canonical_upstream_tarball_basename_is_served() {
.unwrap();
let doc = body_json(served.into_body()).await;
let advertised = doc["versions"]["3001.1.0-exotic"]["dist"]["tarball"].as_str().unwrap();
assert!(advertised.ends_with(&format!("/npm/foo/-/{exotic}")), "got {advertised}");
assert!(advertised.ends_with("/npm/foo@3001.1.0-exotic"), "got {advertised}");
// Fetching `<name>@<version>` or `/-/<exotic>` resolves the upstream
// packument's declared filename and serves the verified bytes.
let version_response = app
.clone()
.oneshot(
Request::get("/npm/foo@3001.1.0-exotic")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(version_response.status(), StatusCode::OK);
assert_eq!(body_bytes(version_response.into_body()).await, bytes);
// And fetching that URL back serves the verified bytes.
let response = app
.oneshot(
Request::get(format!("/npm/foo/-/{exotic}").as_str())
@@ -915,3 +928,82 @@ async fn scoped_tarball_filename_is_canonicalized_before_fetch_and_cache() {
packument_mock.assert_async().await;
mock.assert_async().await;
}
#[tokio::test]
async fn upstream_pnpr_version_route_tarball_is_proxied() {
let mut upstream = mockito::Server::new_async().await;
let bytes = b"chained-pnpr-tarball";
let packument = json!({
"name": "@scope/foo",
"dist-tags": { "latest": "1.0.0" },
"versions": {
"1.0.0": {
"name": "@scope/foo",
"version": "1.0.0",
"dist": {
"tarball": format!("{}/@scope/foo@1.0.0", upstream.url()),
"integrity": sha512_integrity(bytes),
},
},
},
});
let packument_mock = upstream
.mock("GET", "/@scope/foo")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(packument.to_string())
.expect(1)
.create_async()
.await;
let tarball_mock = upstream
.mock("GET", "/@scope/foo@1.0.0")
.with_status(200)
.with_body(bytes)
.expect(1)
.create_async()
.await;
let tmp = TempDir::new().unwrap();
let app = router(config_for(&upstream.url(), tmp.path().to_path_buf()));
let served = app
.clone()
.oneshot(
Request::get("/npm/@scope/foo")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let doc = body_json(served.into_body()).await;
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/@scope/foo@1.0.0",
);
let response = app
.clone()
.oneshot(
Request::get("/npm/@scope/foo@1.0.0")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(body_bytes(response.into_body()).await, bytes);
let filename_response = app
.oneshot(
Request::get("/npm/@scope/foo/-/foo-1.0.0.tgz")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(filename_response.status(), StatusCode::OK);
assert_eq!(body_bytes(filename_response.into_body()).await, bytes);
packument_mock.assert_async().await;
tarball_mock.assert_async().await;
}
@@ -72,7 +72,7 @@ async fn tarball_route_preserves_basename_and_binds_to_declaring_version() {
.unwrap();
assert_eq!(selected.status(), StatusCode::OK);
let selected: Value = serde_json::from_slice(&body_bytes(selected.into_body()).await).unwrap();
assert_eq!(selected["dist"]["tarball"], "http://example.test/npm/foo/-/foo-2.0.0.tgz");
assert_eq!(selected["dist"]["tarball"], "http://example.test/npm/foo@1.0.0");
let full = app
.clone()
@@ -84,14 +84,8 @@ async fn tarball_route_preserves_basename_and_binds_to_declaring_version() {
.await
.unwrap();
let full: Value = serde_json::from_slice(&body_bytes(full.into_body()).await).unwrap();
assert_eq!(
full["versions"]["1.0.0"]["dist"]["tarball"],
"http://example.test/npm/foo/-/foo-2.0.0.tgz",
);
assert_eq!(
full["versions"]["2.0.0"]["dist"]["tarball"],
"http://example.test/npm/foo/-/foo-1.0.0.tgz",
);
assert_eq!(full["versions"]["1.0.0"]["dist"]["tarball"], "http://example.test/npm/foo@1.0.0");
assert_eq!(full["versions"]["2.0.0"]["dist"]["tarball"], "http://example.test/npm/foo@2.0.0");
let route = selected["dist"]["tarball"]
.as_str()
@@ -226,6 +226,10 @@ async fn a_cold_version_request_gets_its_own_tarball() {
let tmp = TempDir::new().unwrap();
let app = router(config_for(&url, tmp.path().to_path_buf()));
let other_response = get(&app, "/npm/foo@2.0.0").await;
assert_eq!(other_response.status(), StatusCode::OK);
assert_eq!(body_bytes(other_response.into_body()).await, other);
let response = get(&app, "/npm/foo@1.0.0").await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(body_bytes(response.into_body()).await, requested);
+34 -4
View File
@@ -3,7 +3,7 @@ pub use discovery::SearchResponse;
pub use packument::{
extract_upstream_revision_integrity, extract_upstream_version_manifest,
extract_version_manifest, rewrite_tarball_urls, rewrite_upstream_tarball_urls,
tarball_basename,
tarball_basename, tarball_basename_for_package, version_route_tarball_version,
};
pub use oci::oci_download_allowed;
@@ -258,15 +258,45 @@ impl Upstream {
name: &CanonicalPackageName,
filename: &str,
) -> Result<FetchOutcome<ThrottledResponse>> {
let url = format!("{}/{}/-/{}", self.base.trim_end_matches('/'), name.as_str(), filename);
self.fetch_tarball_url(&url).await
}
/// Fetch a version-addressed tarball (`{base}/{name}@{version}`) from an
/// upstream `pnpr` registry.
pub async fn fetch_version_tarball_response(
&self,
name: &CanonicalPackageName,
version: &str,
) -> Result<FetchOutcome<ThrottledResponse>> {
let url = format!("{}/{}@{version}", self.base.trim_end_matches('/'), name.as_str());
self.fetch_tarball_url(&url).await
}
/// Fetch a tarball by its version route (`{base}/{name}@{version}`) when
/// the upstream advertised one, or by its archive route
/// (`{base}/{name}/-/{filename}`) otherwise.
pub async fn fetch_tarball_or_version_response(
&self,
name: &CanonicalPackageName,
filename: &str,
version_route: Option<&str>,
) -> Result<FetchOutcome<ThrottledResponse>> {
match version_route {
Some(version) => self.fetch_version_tarball_response(name, version).await,
None => self.fetch_tarball_response(name, filename).await,
}
}
async fn fetch_tarball_url(&self, url: &str) -> Result<FetchOutcome<ThrottledResponse>> {
let started = Instant::now();
self.ensure_available()?;
let url = format!("{}/{}/-/{}", self.base.trim_end_matches('/'), name.as_str(), filename);
let (response, guard) = self.get_with_scoped_headers(&url, &HeaderMap::new()).await?;
let (response, guard) = self.get_with_scoped_headers(url, &HeaderMap::new()).await?;
if response.status() == StatusCode::NOT_FOUND {
self.breaker.record_success();
return Ok(FetchOutcome::NotFound);
}
let response = self.checked(response, &url).await?;
let response = self.checked(response, url).await?;
// Success here covers only headers; a mid-stream body failure is
// the caller's to observe. Recording success on a clean status is
// what verdaccio does too.
+67 -26
View File
@@ -4,21 +4,12 @@ use super::{
};
/// Rewrite every `dist.tarball` in `value` to a URL served by *this*
/// registry instead of whatever URL the source put there. The new URL is
/// `{public_url}/{pkg}/-/{basename}`, where `basename` is the last
/// `/`-separated segment of the original tarball URL. This handles both
/// npm's canonical `/{pkg}/-/{basename}` shape and verdaccio's
/// `/{scope}/{name}/-/{scope}/{filename}` shape uniformly — we only look
/// at the basename, never at the path prefix.
///
/// The basename is preserved verbatim rather than reconstructed from the
/// version, so a non-canonical tarball name (e.g. esprima-fb's zero-padded
/// `esprima-fb-3001.0001.0000-dev-harmony-fb.tgz` for version
/// `3001.1.0-dev-harmony-fb`) survives into the client's lockfile and is
/// fetched back from the path the upstream actually hosts. The tarball
/// endpoint binds each request to a version's declared `dist.integrity`
/// (see `serve_tarball`), so a preserved name can't smuggle in unverified
/// bytes.
/// registry instead of whatever URL the source put there. When the version
/// has an addressable version string, the new URL is `{public_url}/{pkg}@{version}`,
/// and `serve_package_tarball` resolves the version's declared `dist.tarball`
/// filename and `dist.integrity` from its packument or version manifest. When
/// no addressable version is available (such as a top-level manifest without a
/// `version` field), the fallback URL is `{public_url}/{pkg}/-/{basename}`.
///
/// Walks both packument shape (`{ "versions": { v: { dist: ... } } }`)
/// and single-version manifest shape (`{ dist: ... }` at the top level)
@@ -61,18 +52,11 @@ pub(super) fn rewrite_dist_tarball(
) {
// Every string `dist.tarball` must be rewritten to a route on *this*
// server, where integrity and OSV are enforced — never passed through.
// When the upstream URL has no usable basename (e.g. it ends in `/`), fall
// back to the version-derived canonical name (the manifest carries its own
// `version`) so a malformed URL still points at pnpr (and 404s there)
// rather than directing the client at an arbitrary upstream host.
let version = value
.get("version")
.and_then(Value::as_str)
.filter(|version| expected_version.is_none_or(|expected| *version == expected))
.map(str::to_string);
let fallback = version
.as_deref()
.map(|version| pkg.tarball_name_for_version(version));
let Some(dist) = value.get_mut("dist").and_then(Value::as_object_mut) else {
return;
};
@@ -99,11 +83,68 @@ pub(super) fn rewrite_dist_tarball(
dist.remove("revisions");
let Some(tarball_value) = dist.get_mut("tarball") else { return };
let Some(tarball) = tarball_value.as_str() else { return };
let filename = tarball_basename(tarball)
.map(str::to_owned)
.or(fallback)
let target_version = version.as_deref().or(expected_version);
*tarball_value = Value::String(rewritten_tarball_url(pkg, target_version, tarball, public_url));
}
fn rewritten_tarball_url(
pkg: &CanonicalPackageName,
version: Option<&str>,
tarball: &str,
public_url: &str,
) -> String {
if let Some(version) = version
&& is_addressable_version(version)
{
return format!("{public_url}/{pkg}@{version}", pkg = pkg.as_str());
}
let filename = tarball_basename_for_package(tarball, pkg)
.map(std::borrow::Cow::into_owned)
.or_else(|| version.map(|version| pkg.tarball_name_for_version(version)))
.unwrap_or_default();
*tarball_value = Value::String(format!("{public_url}/{}/-/{filename}", pkg.as_str()));
format!("{public_url}/{}/-/{filename}", pkg.as_str())
}
fn is_addressable_version(version: &str) -> bool {
pnpr_package_name::is_safe_path_segment(version)
&& !version
.split_once('+')
.and_then(|(_, build)| build.strip_prefix('r'))
.is_some_and(|digits| {
!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
})
}
/// The `<version>` segment when `url` ends on `{pkg}@{version}`, with any
/// `?query`/`#fragment` stripped, when `<version>` is a safe non-revision
/// version segment.
#[must_use]
pub fn version_route_tarball_version<'a>(
url: &'a str,
pkg: &CanonicalPackageName,
) -> Option<&'a str> {
let path = url
.split(['?', '#'])
.next()
.unwrap_or(url);
let (prefix, version) = path.rsplit_once('@')?;
((prefix.ends_with(&format!("/{}", pkg.as_str())) || prefix == pkg.as_str())
&& is_addressable_version(version))
.then_some(version)
}
/// The tarball filename a `dist.tarball` URL addresses for `pkg`: the canonical
/// `<basename>-<version>.tgz` when `url` is a `{pkg}@{version}` route, or
/// [`tarball_basename`] otherwise.
#[must_use]
pub fn tarball_basename_for_package<'a>(
url: &'a str,
pkg: &CanonicalPackageName,
) -> Option<std::borrow::Cow<'a, str>> {
if let Some(version) = version_route_tarball_version(url, pkg) {
return Some(std::borrow::Cow::Owned(pkg.tarball_name_for_version(version)));
}
tarball_basename(url).map(std::borrow::Cow::Borrowed)
}
fn rewrite_dist_attestations(
@@ -24,10 +24,7 @@ fn rewrites_npm_form_tarball() {
});
let name = CanonicalPackageName::parse("foo", pnpr_package_name::Ecosystem::Npm).unwrap();
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:4873");
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
"http://127.0.0.1:4873/foo/-/foo-1.0.0.tgz",
);
assert_eq!(doc["versions"]["1.0.0"]["dist"]["tarball"], "http://127.0.0.1:4873/foo@1.0.0");
assert_eq!(doc["versions"]["1.0.0"]["dist"]["shasum"], "abc");
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["attestations"]["url"],
@@ -153,7 +150,6 @@ fn does_not_preserve_an_invalid_upstream_revision_route() {
(json!(0), "https://upstream.test/foo@1.0.0+r0"),
(json!(2), "https://other.test/foo@1.0.0+r2"),
] {
let basename = tarball_basename(tarball).unwrap();
let mut doc = json!({
"version": "1.0.0",
"dist": {
@@ -170,7 +166,7 @@ fn does_not_preserve_an_invalid_upstream_revision_route() {
"http://pnpr.test/~corp/",
);
assert_eq!(doc["dist"]["tarball"], format!("http://pnpr.test/~corp/foo/-/{basename}"));
assert_eq!(doc["dist"]["tarball"], "http://pnpr.test/~corp/foo@1.0.0");
assert!(doc["dist"].get("revision").is_none());
}
}
@@ -260,16 +256,35 @@ fn rewrites_verdaccio_form_tarball_for_scoped() {
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:9999");
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
"http://127.0.0.1:9999/@foo/no-deps/-/no-deps-1.0.0.tgz",
"http://127.0.0.1:9999/@foo/no-deps@1.0.0",
);
}
#[test]
fn preserves_non_canonical_tarball_basename() {
// A version whose tarball filename does not follow the
// `<name>-<version>.tgz` convention (here esprima-fb's zero-padded
// name for version `3001.1.0-dev-harmony-fb`) keeps its basename, so
// the client records and fetches back the path the upstream hosts.
fn rewrites_upstream_pnpr_version_route_tarball_url() {
for (pkg, upstream_url, expected) in [
("foo", "https://up.pnpr.test/npm/foo@1.0.0", "http://127.0.0.1:9999/foo@1.0.0"),
(
"@scope/foo",
"https://up.pnpr.test/npm/@scope/foo@1.0.0",
"http://127.0.0.1:9999/@scope/foo@1.0.0",
),
] {
let mut doc = json!({
"versions": {
"1.0.0": {
"dist": { "tarball": upstream_url }
}
}
});
let name = CanonicalPackageName::parse(pkg, pnpr_package_name::Ecosystem::Npm).unwrap();
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:9999");
assert_eq!(doc["versions"]["1.0.0"]["dist"]["tarball"], expected);
}
}
#[test]
fn rewrites_non_canonical_tarball_basename_to_version_route() {
let mut doc = json!({
"versions": {
"3001.1.0-dev-harmony-fb": {
@@ -284,27 +299,28 @@ fn preserves_non_canonical_tarball_basename() {
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:9999");
assert_eq!(
doc["versions"]["3001.1.0-dev-harmony-fb"]["dist"]["tarball"],
"http://127.0.0.1:9999/esprima-fb/-/esprima-fb-3001.0001.0000-dev-harmony-fb.tgz",
"http://127.0.0.1:9999/esprima-fb@3001.1.0-dev-harmony-fb",
);
}
#[test]
fn rewrites_query_bearing_tarball_to_its_path_basename() {
// A signed/query-bearing upstream URL must rewrite to the path basename
// alone: the query is not part of the route a client later requests, so
// keeping it would desync the public URL from the serve-time match.
fn rewrites_query_bearing_tarball_without_query_string() {
let mut doc = json!({
"versions": {
"1.0.0": {
"dist": { "tarball": "https://cdn.example.com/foo/-/foo-1.0.0.tgz?sig=abc&exp=123" }
},
"3001.1.0-dev-harmony-fb": {
"dist": { "tarball": "https://cdn.example.com/foo/-/foo-3001.0001.0000-dev-harmony-fb.tgz?sig=abc&exp=123" }
}
}
});
let name = CanonicalPackageName::parse("foo", pnpr_package_name::Ecosystem::Npm).unwrap();
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:9999");
assert_eq!(doc["versions"]["1.0.0"]["dist"]["tarball"], "http://127.0.0.1:9999/foo@1.0.0");
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
"http://127.0.0.1:9999/foo/-/foo-1.0.0.tgz",
doc["versions"]["3001.1.0-dev-harmony-fb"]["dist"]["tarball"],
"http://127.0.0.1:9999/foo@3001.1.0-dev-harmony-fb",
);
}
@@ -324,10 +340,7 @@ fn rewrites_basenameless_tarball_url_to_pnpr_route() {
});
let name = CanonicalPackageName::parse("foo", pnpr_package_name::Ecosystem::Npm).unwrap();
rewrite_tarball_urls(&mut doc, &name, "http://127.0.0.1:9999");
assert_eq!(
doc["versions"]["1.0.0"]["dist"]["tarball"],
"http://127.0.0.1:9999/foo/-/foo-1.0.0.tgz",
);
assert_eq!(doc["versions"]["1.0.0"]["dist"]["tarball"], "http://127.0.0.1:9999/foo@1.0.0");
}
#[test]
@@ -368,7 +381,7 @@ fn extracts_version_by_dist_tag() {
CanonicalPackageName::parse("@foo/no-deps", pnpr_package_name::Ecosystem::Npm).unwrap();
let manifest = extract_version_manifest(&doc, &name, "latest", "http://reg").unwrap();
assert_eq!(manifest["version"], "1.0.0");
assert_eq!(manifest["dist"]["tarball"], "http://reg/@foo/no-deps/-/no-deps-1.0.0.tgz");
assert_eq!(manifest["dist"]["tarball"], "http://reg/@foo/no-deps@1.0.0");
assert_eq!(manifest["dist"]["shasum"], "abc");
}
@@ -381,7 +394,7 @@ fn extracts_version_by_literal_version() {
let name = CanonicalPackageName::parse("foo", pnpr_package_name::Ecosystem::Npm).unwrap();
let manifest = extract_version_manifest(&doc, &name, "2.0.0", "http://reg").unwrap();
assert_eq!(manifest["version"], "2.0.0");
assert_eq!(manifest["dist"]["tarball"], "http://reg/foo/-/foo-2.0.0.tgz");
assert_eq!(manifest["dist"]["tarball"], "http://reg/foo@2.0.0");
}
#[test]
+5 -2
View File
@@ -75,8 +75,9 @@ Every request routes through the registry graph:
- The serving registry's matching `packages:` rule is enforced on every read
and write.
- Served `dist.tarball` URLs are rewritten onto `--public-url` (`PNPR_PUBLIC_URL`)
and stay canonical for the base the client addressed (`/npm/` or
`/npm/~<name>/`), so lockfiles don't bake in a registry name.
as `<registry>/<name>@<version>` and stay canonical for the base the client
addressed (`/npm/` or `/npm/~<name>/`), so lockfiles don't bake in a
registry name.
## pnpr protocol endpoints
@@ -254,6 +255,8 @@ a hosted registry.
| `PUT` | `/-/pnpm/v1/publish` | pnpm batch publish. Body is `{"packages":[<publish document>, ...]}`. The batch is all-or-nothing and each package requires `publish`. For a release that spans ecosystems, use [`PUT /-/pnpr/v0/publish`](ecosystems.md#one-publish-for-a-workspace-that-spans-ecosystems) instead. |
| `PUT` | `/{package}/-rev/{rev}` | Replace a packument, used by partial unpublish. Requires both `publish` and `unpublish`. Use a percent-encoded scoped package name, for example `@scope%2Fname`. |
| `DELETE` | `/{package}/-rev/{rev}` | Remove an entire package, used by force unpublish. Requires `unpublish`. Use a percent-encoded scoped package name for scoped packages. |
| `DELETE` | `/{name}@{version}/-rev/{rev}` | Remove an unscoped version's tarball. Requires `unpublish`. |
| `DELETE` | `/@scope/{name}@{version}/-rev/{rev}` | Remove a scoped version's tarball. Requires `unpublish`. |
| `DELETE` | `/{name}/-/{filename}/-rev/{rev}` | Remove an unscoped tarball. Requires `unpublish`. |
| `DELETE` | `/@scope/{name}/-/{filename}/-rev/{rev}` | Remove a scoped tarball. Requires `unpublish`. |
| `PUT` | `/-/package/{package}/dist-tags/{tag}` | Add or update a dist-tag. Body is a JSON string version, for example `"1.0.0"`. Requires `publish`. |