mirror of
https://github.com/pnpm/pnpm.git
synced 2026-07-17 11:12:25 -04:00
perf(pnpr): compress semver dependency ranges in abbreviated metadata
Rewrite every semver range in a version's dependencies, peerDependencies, and optionalDependencies to its shortest equivalent form (`^1.0.0`->`1`, `~1.2.0`->`1.2`, `1.2.x`->`1.2`, `^0.5.0`->`0.5`), shrinking the abbreviated packuments pnpr serves on top of the field trimming from #12163. The transform is gated by node-semver's `Range` equality: a `Range` is a normalized `Vec<BoundSet>` deriving `PartialEq`, so `Range::parse(a) == Range::parse(b)` is exact version-set equality. `compress_range` proposes shorter candidates and accepts one only when it re-parses equal to the original, so a value that isn't a plain semver range (git+, npm:alias, workspace:*, dist-tag, tarball URL) or any rewrite that would shift the version set is rejected and the original kept. A caret/x-range upper bound carries a `-0` prerelease floor, so a hand-written explicit `>=x <y` interval is deliberately not collapsed to caret; prerelease-floored ranges are skipped. Compression runs in two places: * `abbreviate_packument` — upstream-proxied packuments are compressed on the fly as they are served. * the publish handler — packages hosted in pnpr are compressed before the manifest is persisted, so they are stored already-compressed and never pay the conversion per read.
This commit is contained in:
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -3219,6 +3219,7 @@ dependencies = [
|
||||
"indexmap",
|
||||
"miette 7.6.0",
|
||||
"mockito",
|
||||
"node-semver",
|
||||
"pacquet-config",
|
||||
"pacquet-config-dir",
|
||||
"pacquet-env-replace",
|
||||
|
||||
@@ -44,6 +44,7 @@ getrandom = { workspace = true }
|
||||
home = { workspace = true }
|
||||
indexmap = { workspace = true }
|
||||
miette = { workspace = true }
|
||||
node-semver = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
rusqlite = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -15,6 +15,7 @@ mod install_accelerator;
|
||||
mod package_name;
|
||||
mod policy;
|
||||
mod publish;
|
||||
mod range_compress;
|
||||
mod search;
|
||||
mod server;
|
||||
mod streaming;
|
||||
|
||||
89
pnpr/crates/pnpr/src/range_compress.rs
Normal file
89
pnpr/crates/pnpr/src/range_compress.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
//! Shrink the abbreviated metadata further by rewriting each semver
|
||||
//! range in a version's dependency maps to the shortest *equivalent*
|
||||
//! string. `^1.0.0` becomes `1`, `~1.2.0` becomes `1.2`,
|
||||
//! `>=1.2.3 <2.0.0` becomes `^1.2.3`, and so on — the resolver sees
|
||||
//! the identical version set, just fewer bytes on the wire.
|
||||
//!
|
||||
//! ## Why this is safe
|
||||
//!
|
||||
//! [`node_semver::Range`] normalizes every range to a canonical set of
|
||||
//! interval bounds and derives [`PartialEq`] over that representation,
|
||||
//! so `Range::parse(a) == Range::parse(b)` is exact version-set
|
||||
//! equality. [`compress_range`] proposes shorter candidate strings and
|
||||
//! only accepts one whose parsed range is `==` the original's. A
|
||||
//! candidate that would shift the version set — or any value that
|
||||
//! isn't a plain semver range at all (a `git+https`, `npm:alias`,
|
||||
//! `workspace:*`, dist-tag, or tarball-URL specifier) — fails the
|
||||
//! check or fails to parse, and the original string is kept. The
|
||||
//! transform can only ever make a range shorter, never change what it
|
||||
//! resolves to.
|
||||
|
||||
use node_semver::Range;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Version-object fields whose values are semver ranges keyed by
|
||||
/// dependency name. `bundleDependencies` (an array of names) and
|
||||
/// `peerDependenciesMeta` (per-dep flags) carry no ranges, so they're
|
||||
/// excluded.
|
||||
const DEPENDENCY_FIELDS: &[&str] = &["dependencies", "peerDependencies", "optionalDependencies"];
|
||||
|
||||
/// Compress the dependency ranges of every version in a full packument
|
||||
/// in place. Used at publish time so packages hosted in pnpr are
|
||||
/// stored already-compressed.
|
||||
pub fn compress_packument_dependencies(packument: &mut Value) {
|
||||
let Some(versions) = packument.get_mut("versions").and_then(Value::as_object_mut) else {
|
||||
return;
|
||||
};
|
||||
for version in versions.values_mut() {
|
||||
compress_version_dependencies(version);
|
||||
}
|
||||
}
|
||||
|
||||
/// Compress the dependency ranges of a single version object in place.
|
||||
pub fn compress_version_dependencies(version: &mut Value) {
|
||||
let Some(obj) = version.as_object_mut() else { return };
|
||||
for &field in DEPENDENCY_FIELDS {
|
||||
let Some(deps) = obj.get_mut(field).and_then(Value::as_object_mut) else {
|
||||
continue;
|
||||
};
|
||||
for spec in deps.values_mut() {
|
||||
let Some(range) = spec.as_str() else { continue };
|
||||
if let Some(shorter) = compress_range(range) {
|
||||
*spec = Value::String(shorter);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a strictly-shorter string denoting the identical version set
|
||||
/// as `range`, or `None` when no shorter equivalent is found (or
|
||||
/// `range` isn't a parseable semver range). Every returned candidate
|
||||
/// is verified to parse back to a range `==` the original, so the
|
||||
/// result is always a safe, drop-in replacement.
|
||||
pub fn compress_range(range: &str) -> Option<String> {
|
||||
let parsed = Range::parse(range).ok()?;
|
||||
|
||||
// The re-serialized canonical form normalizes whitespace and
|
||||
// comparator order for free; the explicit caret/tilde/partial
|
||||
// forms below capture the reductions Display doesn't perform.
|
||||
let mut candidates = vec![parsed.to_string()];
|
||||
if let Some(min) = parsed.min_version()
|
||||
&& !min.is_prerelease()
|
||||
{
|
||||
let (major, minor, patch) = (min.major, min.minor, min.patch);
|
||||
candidates.push(format!("{major}"));
|
||||
candidates.push(format!("{major}.{minor}"));
|
||||
candidates.push(format!("{major}.{minor}.{patch}"));
|
||||
candidates.push(format!("^{major}.{minor}.{patch}"));
|
||||
candidates.push(format!("~{major}.{minor}.{patch}"));
|
||||
}
|
||||
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|candidate| candidate.len() < range.len())
|
||||
.filter(|candidate| Range::parse(candidate).is_ok_and(|reparsed| reparsed == parsed))
|
||||
.min_by_key(String::len)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
164
pnpr/crates/pnpr/src/range_compress/tests.rs
Normal file
164
pnpr/crates/pnpr/src/range_compress/tests.rs
Normal file
@@ -0,0 +1,164 @@
|
||||
use super::*;
|
||||
use node_semver::Version;
|
||||
use serde_json::json;
|
||||
|
||||
/// Assert a range compresses to `expected` and that the compressed
|
||||
/// form denotes the exact same version set as the original under
|
||||
/// node-semver (the same engine pnpm/pacquet resolve with).
|
||||
fn assert_compresses(original: &str, expected: &str) {
|
||||
let got = compress_range(original);
|
||||
assert_eq!(got.as_deref(), Some(expected), "compress_range({original:?})");
|
||||
assert_equivalent(original, expected);
|
||||
}
|
||||
|
||||
/// Assert a range is left untouched (no strictly-shorter equivalent).
|
||||
fn assert_unchanged(original: &str) {
|
||||
assert_eq!(compress_range(original), None, "compress_range({original:?}) should be a no-op");
|
||||
}
|
||||
|
||||
fn assert_equivalent(left: &str, right: &str) {
|
||||
assert_eq!(
|
||||
Range::parse(left).unwrap(),
|
||||
Range::parse(right).unwrap(),
|
||||
"{left:?} and {right:?} must denote the same version set",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caret_with_zero_minor_and_patch_becomes_bare_major() {
|
||||
assert_compresses("^1.0.0", "1");
|
||||
assert_compresses("^12.0.0", "12");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caret_on_zero_major_becomes_major_minor() {
|
||||
// `^0.5.0` == `>=0.5.0 <0.6.0` == `0.5`.
|
||||
assert_compresses("^0.5.0", "0.5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tilde_with_zero_patch_becomes_major_minor() {
|
||||
assert_compresses("~1.2.0", "1.2");
|
||||
assert_compresses("~1.0.0", "1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x_ranges_collapse_to_partials() {
|
||||
assert_compresses("1.x", "1");
|
||||
assert_compresses("1.x.x", "1");
|
||||
assert_compresses("1.2.x", "1.2");
|
||||
assert_compresses("1.2.X", "1.2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hand_written_explicit_intervals_are_not_rewritten_to_caret() {
|
||||
// A caret/x-range's upper bound is `<2.0.0-0` (it excludes
|
||||
// prereleases of the next major), so it is *not* the same version
|
||||
// set as a hand-written `<2.0.0`. node-semver — and JS semver —
|
||||
// distinguish the two, so the conversion is rejected and the
|
||||
// canonical (whitespace-normalized) form is kept instead.
|
||||
assert_unchanged(">=1.0.0 <2.0.0");
|
||||
assert_compresses(">=1.2.3 <2.0.0", ">=1.2.3 <2.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_is_normalized() {
|
||||
assert_compresses(">= 1.2.3 < 2.0.0", ">=1.2.3 <2.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_minimal_ranges_are_left_alone() {
|
||||
// Nonzero patch under a caret can't shrink to a partial.
|
||||
assert_unchanged("^1.2.3");
|
||||
// Caret-0 with a nonzero patch is a one-patch window; irreducible.
|
||||
assert_unchanged("^0.0.3");
|
||||
// Bare partials and wildcards are already shortest.
|
||||
assert_unchanged("1");
|
||||
assert_unchanged("1.2");
|
||||
assert_unchanged("*");
|
||||
// A pinned exact version is already as short as it gets.
|
||||
assert_unchanged("1.2.3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_semver_specifiers_pass_through_untouched() {
|
||||
assert_unchanged("workspace:*");
|
||||
assert_unchanged("npm:other@^1.0.0");
|
||||
assert_unchanged("git+https://example.com/a/b.git");
|
||||
assert_unchanged("file:../local");
|
||||
assert_unchanged("latest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prerelease_ranges_are_not_rewritten() {
|
||||
// Prerelease handling is the one area where the JS/Rust engines
|
||||
// can diverge, so we conservatively leave these alone.
|
||||
assert_unchanged("^1.2.3-alpha.1 <2.0.0");
|
||||
assert_unchanged(">=1.2.3-rc.1 <2.0.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compression_is_idempotent() {
|
||||
let once = compress_range("^1.0.0").unwrap();
|
||||
assert_eq!(once, "1");
|
||||
assert_eq!(compress_range(&once), None);
|
||||
}
|
||||
|
||||
/// The compressed form must accept and reject exactly the versions the
|
||||
/// original does at the interval boundaries.
|
||||
#[test]
|
||||
fn boundary_versions_match_after_compression() {
|
||||
let cases = [("^1.0.0", "1"), ("1.2.x", "1.2"), ("~1.2.0", "1.2")];
|
||||
let probes = ["0.9.9", "1.0.0", "1.2.2", "1.2.3", "1.2.4", "1.9.9", "2.0.0"];
|
||||
for (original, _expected) in cases {
|
||||
let before = Range::parse(original).unwrap();
|
||||
let compressed = compress_range(original).unwrap();
|
||||
let after = Range::parse(&compressed).unwrap();
|
||||
for probe in probes {
|
||||
let version = Version::parse(probe).unwrap();
|
||||
assert_eq!(
|
||||
before.satisfies(&version),
|
||||
after.satisfies(&version),
|
||||
"{original:?} vs {compressed:?} disagree on {probe}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_version_dependencies_rewrites_every_map() {
|
||||
let mut version = json!({
|
||||
"name": "foo",
|
||||
"version": "1.0.0",
|
||||
"dependencies": { "bar": "^1.0.0", "qux": "git+https://x/y.git" },
|
||||
"peerDependencies": { "react": "^16.0.0" },
|
||||
"optionalDependencies": { "fsevents": "~2.3.0" },
|
||||
"peerDependenciesMeta": { "react": { "optional": true } },
|
||||
});
|
||||
|
||||
compress_version_dependencies(&mut version);
|
||||
|
||||
assert_eq!(version["dependencies"]["bar"], "1");
|
||||
// Non-semver specifier untouched.
|
||||
assert_eq!(version["dependencies"]["qux"], "git+https://x/y.git");
|
||||
assert_eq!(version["peerDependencies"]["react"], "16");
|
||||
assert_eq!(version["optionalDependencies"]["fsevents"], "2.3");
|
||||
// Non-range maps are left alone.
|
||||
assert_eq!(version["peerDependenciesMeta"]["react"]["optional"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_packument_dependencies_walks_all_versions() {
|
||||
let mut packument = json!({
|
||||
"name": "foo",
|
||||
"versions": {
|
||||
"1.0.0": { "dependencies": { "bar": "^1.0.0" } },
|
||||
"2.0.0": { "dependencies": { "bar": "2.x" } },
|
||||
}
|
||||
});
|
||||
|
||||
compress_packument_dependencies(&mut packument);
|
||||
|
||||
assert_eq!(packument["versions"]["1.0.0"]["dependencies"]["bar"], "1");
|
||||
assert_eq!(packument["versions"]["2.0.0"]["dependencies"]["bar"], "2");
|
||||
}
|
||||
@@ -9,6 +9,7 @@ use crate::{
|
||||
PendingAttachment, extract_attachments, iso_from_unix_millis, merge_manifest, now_iso,
|
||||
stream_decode_verify_and_write,
|
||||
},
|
||||
range_compress::compress_packument_dependencies,
|
||||
streaming,
|
||||
upstream::{
|
||||
FetchOutcome, Upstream, abbreviate_packument, extract_version_manifest,
|
||||
@@ -826,6 +827,11 @@ async fn publish_package(
|
||||
Err(err) => return error_response(&err),
|
||||
};
|
||||
|
||||
// Rewrite dependency ranges to their shortest equivalent form
|
||||
// before the manifest is persisted, so packages hosted in pnpr are
|
||||
// stored already-compressed and never pay the conversion per read.
|
||||
compress_packument_dependencies(&mut incoming);
|
||||
|
||||
// Resolve each attachment's canonical disk filename + matching
|
||||
// `versions[v].dist` block. Attachment names that don't match the
|
||||
// package (`bar-1.0.0.tgz` for `foo`) or that try to escape the
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::{
|
||||
error::{RegistryError, Result},
|
||||
package_name::PackageName,
|
||||
range_compress::compress_version_dependencies,
|
||||
};
|
||||
use pacquet_network::ThrottledClient;
|
||||
use reqwest::StatusCode;
|
||||
@@ -183,7 +184,9 @@ const ABBREVIATED_VERSION_FIELDS: &[&str] = &[
|
||||
|
||||
/// Strip a parsed packument down to the abbreviated install-v1 form.
|
||||
/// Should be called *after* `rewrite_tarball_urls` so the returned
|
||||
/// document's `dist.tarball` URLs already point at this server.
|
||||
/// document's `dist.tarball` URLs already point at this server. Each
|
||||
/// version's dependency ranges are rewritten to their shortest
|
||||
/// equivalent form by [`compress_version_dependencies`].
|
||||
pub fn abbreviate_packument(packument: &Value) -> Value {
|
||||
let mut out = serde_json::Map::new();
|
||||
if let Some(obj) = packument.as_object() {
|
||||
@@ -209,7 +212,9 @@ pub fn abbreviate_packument(packument: &Value) -> Value {
|
||||
}
|
||||
}
|
||||
trim_dist_fields(&mut trimmed);
|
||||
abbreviated_versions.insert(version_id.clone(), Value::Object(trimmed));
|
||||
let mut trimmed = Value::Object(trimmed);
|
||||
compress_version_dependencies(&mut trimmed);
|
||||
abbreviated_versions.insert(version_id.clone(), trimmed);
|
||||
}
|
||||
out.insert("versions".to_string(), Value::Object(abbreviated_versions));
|
||||
}
|
||||
|
||||
@@ -149,7 +149,9 @@ fn abbreviation_drops_fields_the_resolver_ignores() {
|
||||
let version = &out["versions"]["1.0.0"];
|
||||
// Resolver-relevant fields kept.
|
||||
assert_eq!(version["name"], "foo");
|
||||
assert_eq!(version["dependencies"]["bar"], "^1.0.0");
|
||||
// Dependency ranges are compressed to their shortest equivalent:
|
||||
// `^1.0.0` denotes the same version set as `1`.
|
||||
assert_eq!(version["dependencies"]["bar"], "1");
|
||||
assert_eq!(version["peerDependencies"]["react"], "*");
|
||||
assert_eq!(version["hasInstallScript"], true);
|
||||
// Platform-filtering fields kept for optional-dep selection (`#9950`).
|
||||
|
||||
@@ -202,6 +202,43 @@ async fn authenticated_publish_writes_manifest_and_tarball() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publish_stores_dependency_ranges_in_compressed_form() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let storage = tmp.path().to_path_buf();
|
||||
let app = router(static_config(storage.clone()));
|
||||
let (app, token) = add_user_and_get_token(app, "alice", "secret").await;
|
||||
|
||||
let bytes = b"fake-tarball-bytes";
|
||||
let mut body = sample_publish_body("deppkg", "1.0.0", bytes);
|
||||
body["versions"]["1.0.0"]["dependencies"] = json!({
|
||||
"caret": "^1.0.0",
|
||||
"tilde": "~2.3.0",
|
||||
"wildcard": "3.x",
|
||||
"pinned": "4.5.6",
|
||||
"git": "git+https://example.com/a/b.git",
|
||||
});
|
||||
let request = Request::put("/deppkg")
|
||||
.header("content-type", "application/json")
|
||||
.header("Authorization", format!("Bearer {token}"))
|
||||
.body(Body::from(serde_json::to_vec(&body).unwrap()))
|
||||
.unwrap();
|
||||
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
|
||||
let on_disk = std::fs::read(storage.join("deppkg/package.json")).expect("packument written");
|
||||
let packument: Value = serde_json::from_slice(&on_disk).unwrap();
|
||||
let deps = &packument["versions"]["1.0.0"]["dependencies"];
|
||||
assert_eq!(deps["caret"], "1");
|
||||
assert_eq!(deps["tilde"], "2.3");
|
||||
assert_eq!(deps["wildcard"], "3");
|
||||
// A pinned exact version is already minimal; a non-semver
|
||||
// specifier is left untouched.
|
||||
assert_eq!(deps["pinned"], "4.5.6");
|
||||
assert_eq!(deps["git"], "git+https://example.com/a/b.git");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publish_followed_by_dist_tag_set_works() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user