diff --git a/.changeset/pnpr-readable-store-paths.md b/.changeset/pnpr-readable-store-paths.md new file mode 100644 index 0000000000..811a97e79c --- /dev/null +++ b/.changeset/pnpr-readable-store-paths.md @@ -0,0 +1,7 @@ +--- +"@pnpm/pnpr": minor +--- + +pnpr stores packages under readable paths. `@scope/pkg` is stored in `packages/@scope/pkg/`, an unscoped package in `packages/@//`, and a tarball as `blobs/.tgz`. + +A store written by an earlier pnpr is not read. Publish its packages again into an empty store. diff --git a/pnpm/tasks/integrated-benchmark/src/work_env/tests.rs b/pnpm/tasks/integrated-benchmark/src/work_env/tests.rs index 7bbd9b235d..e0f6216065 100644 --- a/pnpm/tasks/integrated-benchmark/src/work_env/tests.rs +++ b/pnpm/tasks/integrated-benchmark/src/work_env/tests.rs @@ -130,8 +130,14 @@ fn peer_heavy_scenario_generates_shared_subgraph_root() { .expect("read leaf peer-heavy packument"), ) .expect("parse leaf peer-heavy packument"); - let registry_package_count = - fs::read_dir(registry.join("default/packages")).expect("read peer-heavy registry").count(); + let registry_package_count: usize = fs::read_dir(registry.join("default/packages")) + .expect("read peer-heavy registry") + .map(|scope| { + fs::read_dir(scope.expect("read peer-heavy registry scope").path()) + .expect("read peer-heavy registry scope") + .count() + }) + .sum(); let _ = fs::remove_dir_all(&dir); let dependencies = manifest["dependencies"].as_object().expect("generated dependencies"); diff --git a/pnpr/crates/pnpr-fixtures/src/lib.rs b/pnpr/crates/pnpr-fixtures/src/lib.rs index ed3da98f25..6210db12cc 100644 --- a/pnpr/crates/pnpr-fixtures/src/lib.rs +++ b/pnpr/crates/pnpr-fixtures/src/lib.rs @@ -250,6 +250,11 @@ fn fixture_fingerprint(root: &Path) -> String { let mut hasher = Sha256::new(); // A storage tree built in another layout must not be reused. hasher.update(pnpr_storage::layout::LAYOUT_VERSION.to_le_bytes()); + hasher.update( + local_blob_path(Path::new(""), "", "@scope/name", "name-1.0.0.tgz") + .to_string_lossy() + .as_bytes(), + ); for entry in fixture_files(root) { let path = entry.path(); let relative = path.strip_prefix(root).expect("fixture entry under root"); diff --git a/pnpr/crates/pnpr/src/upgrade_storage/plan.rs b/pnpr/crates/pnpr/src/upgrade_storage/plan.rs index c83a2d3e0c..77f0b1c91f 100644 --- a/pnpr/crates/pnpr/src/upgrade_storage/plan.rs +++ b/pnpr/crates/pnpr/src/upgrade_storage/plan.rs @@ -7,7 +7,10 @@ //! `package.json` was a package: its other files were its blobs, and its //! subdirectories other packages (nested image repositories). -use pnpr_storage::layout::{decode_segment, encode_segment, namespace_segments}; +use pnpr_storage::layout::{ + blob_segment, decode_blob_segment, decode_package_segments, encode_segment, namespace_segments, + package_segments, +}; use std::collections::{BTreeMap, BTreeSet}; const LEGACY_DOCUMENT: &str = "package.json"; @@ -157,10 +160,12 @@ fn is_new_namespace_path(rest: &str) -> bool { fn is_new_package_path(rest: &str) -> bool { let components: Vec<&str> = rest.split('/').collect(); - let decodes = |segment: &str| decode_segment(segment).is_some(); match components.as_slice() { - ["packages", package, "document.json"] => decodes(package), - ["packages", package, "blobs", blob] => decodes(package) && decodes(blob), + ["packages", scope, package, "document.json"] => { + decode_package_segments(scope, package).is_some() + } + ["packages", scope, package, "blobs", blob] => decode_package_segments(scope, package) + .is_some_and(|name| decode_blob_segment(&name, blob).is_some()), _ => false, } } @@ -229,11 +234,14 @@ fn package_step(prefix: &str, rest: &str, is_package: impl Fn(&str) -> bool) -> if !is_package(package) || file.starts_with('.') { return Step::Orphan; } - let package_dir = format!("{prefix}/packages/{}", encode_segment(package)); + let package_dir = format!("{prefix}/packages/{}", package_segments(package).join("/")); if file == LEGACY_DOCUMENT { Step::Move { to: format!("{package_dir}/document.json"), document: true } } else { - Step::Move { to: format!("{package_dir}/blobs/{}", encode_segment(file)), document: false } + Step::Move { + to: format!("{package_dir}/blobs/{}", blob_segment(package, file)), + document: false, + } } } diff --git a/pnpr/crates/pnpr/src/upgrade_storage/plan/tests.rs b/pnpr/crates/pnpr/src/upgrade_storage/plan/tests.rs index 6bda89b1a3..fab4f07bea 100644 --- a/pnpr/crates/pnpr/src/upgrade_storage/plan/tests.rs +++ b/pnpr/crates/pnpr/src/upgrade_storage/plan/tests.rs @@ -34,28 +34,25 @@ fn packages_without_an_org_move_into_the_default_namespace() { &[], ), [ - ("left-pad/package.json", moved("default/packages/left-pad/document.json", true)), + ("left-pad/package.json", moved("default/packages/@/left-pad/document.json", true)), ( "left-pad/left-pad-1.0.0.tgz", - moved("default/packages/left-pad/blobs/left-pad-1.0.0.tgz", false) - ), - ( - "@scope/App/package.json", - moved("default/packages/!40scope!2f!41pp/document.json", true) + moved("default/packages/@/left-pad/blobs/1.0.0.tgz", false) ), + ("@scope/App/package.json", moved("default/packages/@scope/!41pp/document.json", true)), ( "@scope/App/App-1.0.0.tgz", - moved("default/packages/!40scope!2f!41pp/blobs/!41pp-1.0.0.tgz", false), + moved("default/packages/@scope/!41pp/blobs/1.0.0.tgz", false), ), - ("acme/app/package.json", moved("default/packages/acme!2fapp/document.json", true)), - ("acme/app/sha256-aa", moved("default/packages/acme!2fapp/blobs/sha256-aa", false)), + ("acme/app/package.json", moved("default/packages/@/acme!2fapp/document.json", true)), + ("acme/app/sha256-aa", moved("default/packages/@/acme!2fapp/blobs/sha256-aa", false)), ( "acme/app/tool/package.json", - moved("default/packages/acme!2fapp!2ftool/document.json", true) + moved("default/packages/@/acme!2fapp!2ftool/document.json", true) ), ( "acme/app/tool/sha256-bb", - moved("default/packages/acme!2fapp!2ftool/blobs/sha256-bb", false) + moved("default/packages/@/acme!2fapp!2ftool/blobs/sha256-bb", false) ), ] .map(|(from, step)| (from.to_string(), step)), @@ -78,11 +75,11 @@ fn an_org_directory_becomes_its_namespace() { [ ( "npm~internal/tool/package.json", - moved("orgs/npm!7einternal/packages/tool/document.json", true) + moved("orgs/npm!7einternal/packages/@/tool/document.json", true) ), ( "npm~internal/tool/tool-1.0.0.tgz", - moved("orgs/npm!7einternal/packages/tool/blobs/tool-1.0.0.tgz", false), + moved("orgs/npm!7einternal/packages/@/tool/blobs/1.0.0.tgz", false), ), ("npm~internal/.revisions/sha512/abc/index.json", Step::Remove,), ( @@ -138,7 +135,7 @@ fn tar_integrities_keep_their_namespace() { steps(&[".tar-integrity/@scope/pkg/package.json"], &[]), [( ".tar-integrity/@scope/pkg/package.json".to_string(), - moved(".tar-integrity/packages/!40scope!2fpkg/document.json", true), + moved(".tar-integrity/packages/@scope/pkg/document.json", true), )], ); } @@ -152,7 +149,7 @@ fn files_outside_any_package_are_orphans() { ), [ ("stray/file.tgz", Step::Orphan), - ("left-pad/package.json", moved("default/packages/left-pad/document.json", true)), + ("left-pad/package.json", moved("default/packages/@/left-pad/document.json", true)), ("left-pad/.left-pad-1.0.0.tgz.tmp", Step::Orphan), ] .map(|(from, step)| (from.to_string(), step)), @@ -164,16 +161,16 @@ fn an_interrupted_upgrade_resumes_where_it_stopped() { assert_eq!( steps( &[ - "default/packages/left-pad/blobs/left-pad-1.0.0.tgz", + "default/packages/@/left-pad/blobs/1.0.0.tgz", "left-pad/package.json", - "orgs/acme/packages/tool/document.json", - ".tar-integrity/packages/!40scope!2fpkg/document.json", + "orgs/acme/packages/@/tool/document.json", + ".tar-integrity/packages/@scope/pkg/document.json", ], &["acme"], ), [( "left-pad/package.json".to_string(), - moved("default/packages/left-pad/document.json", true) + moved("default/packages/@/left-pad/document.json", true) )], ); } diff --git a/pnpr/crates/pnpr/src/upgrade_storage/tests.rs b/pnpr/crates/pnpr/src/upgrade_storage/tests.rs index 3b51638af6..9ebee4ec92 100644 --- a/pnpr/crates/pnpr/src/upgrade_storage/tests.rs +++ b/pnpr/crates/pnpr/src/upgrade_storage/tests.rs @@ -100,7 +100,7 @@ async fn an_interrupted_local_conversion_resumes() { let tmp = TempDir::new().unwrap(); let root = tmp.path().join("storage"); write_legacy(&root).await; - let moved = root.join("default/packages/left-pad/blobs/left-pad-1.0.0.tgz"); + let moved = root.join("default/packages/@/left-pad/blobs/1.0.0.tgz"); tokio::fs::create_dir_all(moved.parent().unwrap()).await.unwrap(); tokio::fs::rename(root.join("left-pad/left-pad-1.0.0.tgz"), &moved).await.unwrap(); let storage = diff --git a/pnpr/crates/pnpr/tests/cache_eviction.rs b/pnpr/crates/pnpr/tests/cache_eviction.rs index 5c59ccae48..df1a045c69 100644 --- a/pnpr/crates/pnpr/tests/cache_eviction.rs +++ b/pnpr/crates/pnpr/tests/cache_eviction.rs @@ -8,11 +8,11 @@ use pnpr::{ prune_orphaned_namespaces_startup, }, }; -use pnpr_storage::layout::encode_segment; +use pnpr_storage::layout::package_segments; use std::{ fs::{File, FileTimes}, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - path::Path, + path::{Path, PathBuf}, time::{Duration, SystemTime}, }; use tempfile::TempDir; @@ -224,7 +224,7 @@ async fn ttl_eviction_prunes_empty_package_directories() { .path() .join(active_ns) .join("packages") - .join(encode_segment("@my-scope/my-pkg")) + .join(PathBuf::from_iter(package_segments("@my-scope/my-pkg"))) .join("document.json"); create_file_with_age(&scoped_file, b"{}", Duration::from_hours(240)); @@ -241,7 +241,7 @@ async fn ttl_eviction_prunes_empty_package_directories() { !tmp.path() .join(active_ns) .join("packages") - .join(encode_segment("@my-scope/my-pkg")) + .join(PathBuf::from_iter(package_segments("@my-scope/my-pkg"))) .exists(), ); assert!( diff --git a/pnpr/crates/pnpr/tests/cargo_registry/routing.rs b/pnpr/crates/pnpr/tests/cargo_registry/routing.rs index af348d4b7b..f3654f115c 100644 --- a/pnpr/crates/pnpr/tests/cargo_registry/routing.rs +++ b/pnpr/crates/pnpr/tests/cargo_registry/routing.rs @@ -4,6 +4,7 @@ use super::{ publish_request, router_with_auth, sha256_hex, }; use crate::tokens::issue_token; +use pnpr_storage::layout::blob_segment; #[tokio::test] async fn config_json_points_downloads_and_the_api_back_at_the_registry() { @@ -309,7 +310,10 @@ async fn proxies_the_sparse_index_and_verified_downloads_through_an_upstream() { index_mock.assert_async().await; download_mock.assert_async().await; let cache = tmp.path().join(".pnpr-cache"); - assert!(find_file(&cache, "serde-1.0.0.crate").is_some(), "download is cached"); + assert!( + find_file(&cache, &blob_segment("serde", "serde-1.0.0.crate")).is_some(), + "download is cached", + ); // An unknown crate is a definitive 404, and the cache holds nothing for it. let missing = upstream @@ -374,7 +378,10 @@ async fn a_download_that_fails_the_index_checksum_is_never_cached() { .await .unwrap(); assert!(axum::body::to_bytes(response.into_body(), usize::MAX).await.is_err()); - assert!(find_file(&tmp.path().join(".pnpr-cache"), "serde-1.0.0.crate").is_none()); + assert!( + find_file(&tmp.path().join(".pnpr-cache"), &blob_segment("serde", "serde-1.0.0.crate")) + .is_none(), + ); } #[tokio::test] diff --git a/pnpr/crates/pnpr/tests/oci_registry/manifests.rs b/pnpr/crates/pnpr/tests/oci_registry/manifests.rs index 84752998e7..a4b8c9778e 100644 --- a/pnpr/crates/pnpr/tests/oci_registry/manifests.rs +++ b/pnpr/crates/pnpr/tests/oci_registry/manifests.rs @@ -397,7 +397,7 @@ async fn referrer_migration_does_not_block_writers_or_restore_deleted_manifests( }) .await .unwrap(); - objects.pause(digest.blob_filename()); + objects.pause(pnpr_storage::layout::blob_segment(key.as_str(), &digest.blob_filename())); let reader = app.clone(); let path = format!("/v2/acme/migration/referrers/{subject}"); let read = tokio::spawn(async move { get(&reader, &path).await }); diff --git a/pnpr/crates/pnpr/tests/pypi_registry.rs b/pnpr/crates/pnpr/tests/pypi_registry.rs index 3bb2895d22..a06a13f443 100644 --- a/pnpr/crates/pnpr/tests/pypi_registry.rs +++ b/pnpr/crates/pnpr/tests/pypi_registry.rs @@ -9,7 +9,7 @@ use axum::{ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; use common::{HostedSource, PUBLIC_URL, body_bytes, find_file, mixed_router_config, sha256_hex}; use pnpr::{AuthState, Config, Ecosystem, recover_publish_journal, router_with_auth}; -use pnpr_storage::layout::{local_blob_path, write_local_layout_marker}; +use pnpr_storage::layout::{blob_segment, local_blob_path, write_local_layout_marker}; use serde_json::{Value, json}; use std::path::{Path, PathBuf}; use tempfile::TempDir; @@ -434,7 +434,10 @@ async fn proxies_a_simple_page_and_verified_downloads_through_an_upstream() { } page_mock.assert_async().await; file_mock.assert_async().await; - assert!(find_file(&tmp.path().join(".pnpr-cache"), filename).is_some(), "file is cached"); + assert!( + find_file(&tmp.path().join(".pnpr-cache"), &blob_segment("requests", filename)).is_some(), + "file is cached", + ); // A file the page does not list is never fetched. let response = app @@ -507,7 +510,7 @@ async fn an_upstream_without_the_json_api_is_a_gateway_error_and_a_bad_hash_is_n .await .unwrap(); assert!(axum::body::to_bytes(response.into_body(), usize::MAX).await.is_err()); - assert!(find_file(&tmp.path().join(".pnpr-cache"), filename).is_none()); + assert!(find_file(&tmp.path().join(".pnpr-cache"), &blob_segment("lying", filename)).is_none()); } #[tokio::test] diff --git a/pnpr/crates/pnpr/tests/server.rs b/pnpr/crates/pnpr/tests/server.rs index 8ac243a673..3246bdf584 100644 --- a/pnpr/crates/pnpr/tests/server.rs +++ b/pnpr/crates/pnpr/tests/server.rs @@ -69,7 +69,7 @@ use pnpr::{ PackageRule, PackageRules, PublicRoute, Registries, Registry, router, router_with_auth, }; use pnpr_fixtures::MockConfigPresets; -use pnpr_storage::layout::{encode_segment, local_document_path}; +use pnpr_storage::layout::{blob_segment, local_document_path, package_segments}; use serde_json::{Value, json}; use ssri::{Algorithm, IntegrityOpts}; use std::{ @@ -150,7 +150,7 @@ fn public_cache_pkg(cache_root: &Path, pkg: &str) -> PathBuf { .cloned() .unwrap_or_else(|| public.join("__none__")) .join("packages") - .join(encode_segment(pkg)) + .join(PathBuf::from_iter(package_segments(pkg))) } /// The directory [`public_cache_pkg`] keeps `pkg`'s cached tarballs in. @@ -158,6 +158,11 @@ fn public_cache_blobs(cache_root: &Path, pkg: &str) -> PathBuf { public_cache_pkg(cache_root, pkg).join("blobs") } +/// Where [`public_cache_blobs`] keeps `pkg`'s cached `file`. +fn public_cache_blob(cache_root: &Path, pkg: &str, file: &str) -> PathBuf { + public_cache_blobs(cache_root, pkg).join(blob_segment(pkg, file)) +} + async fn body_bytes(body: Body) -> Vec { to_bytes(body, usize::MAX).await.expect("read body").to_vec() } diff --git a/pnpr/crates/pnpr/tests/server/streaming_cache.rs b/pnpr/crates/pnpr/tests/server/streaming_cache.rs index 02da87d55b..1f49b66025 100644 --- a/pnpr/crates/pnpr/tests/server/streaming_cache.rs +++ b/pnpr/crates/pnpr/tests/server/streaming_cache.rs @@ -1,7 +1,7 @@ use super::{ Body, Duration, Request, ServiceExt, StatusCode, TempDir, await_no_tgz, body_bytes, config_for, - json, mock_packument_for_tarball, public_cache_blobs, public_cache_pkg, router, - sha512_integrity, spawn_truncated_upstream, tarball_cache_entries, + json, mock_packument_for_tarball, public_cache_blob, public_cache_blobs, public_cache_pkg, + router, sha512_integrity, spawn_truncated_upstream, tarball_cache_entries, }; /// Stale-if-error: once a cached packument is stale, a refetch that fails @@ -181,7 +181,7 @@ async fn concurrent_tarball_fetches_settle_to_one_cache_file() { // path, so one tarball remains with no temporary siblings. let dir = public_cache_blobs(&cache_dir, "foo"); let entries = tarball_cache_entries(&dir); - assert_eq!(entries, vec!["foo-1.0.0.tgz".to_string()]); + assert_eq!(entries, vec!["1.0.0.tgz".to_string()]); mock.assert_async().await; } @@ -218,7 +218,7 @@ async fn cache_tmp_open_failure_fails_closed() { assert_eq!(response.status(), StatusCode::BAD_GATEWAY); // The cache path under the not-a-dir should not exist. - let cache_path = public_cache_blobs(&blocked, "foo").join("foo-1.0.0.tgz"); + let cache_path = public_cache_blob(&blocked, "foo", "foo-1.0.0.tgz"); assert!(!cache_path.exists()); } @@ -305,7 +305,7 @@ async fn proxied_tarball_streams_to_client_and_is_cached() { // end-of-stream SRI check that promotes the verified bytes to the cache. assert_eq!(body_bytes(response.into_body()).await, bytes); - let cache_path = public_cache_blobs(&cache_dir, "big").join("big-1.0.0.tgz"); + let cache_path = public_cache_blob(&cache_dir, "big", "big-1.0.0.tgz"); assert_eq!(std::fs::read(cache_path).unwrap(), bytes); } diff --git a/pnpr/crates/pnpr/tests/server/tar_registry.rs b/pnpr/crates/pnpr/tests/server/tar_registry.rs index 4a2e2b1679..12e3c77f75 100644 --- a/pnpr/crates/pnpr/tests/server/tar_registry.rs +++ b/pnpr/crates/pnpr/tests/server/tar_registry.rs @@ -8,7 +8,7 @@ use super::{ sha512_integrity, tar_of, }; use crate::tokens::issue_token; -use pnpr_storage::layout::encode_segment; +use pnpr_storage::layout::package_segments; fn widget_tar() -> Vec { tar_of(&[("package/package.json", br#"{"name":"@acme/widget","version":"1.0.0"}"#)]) @@ -194,7 +194,7 @@ async fn an_upstream_version_gains_its_tar_integrity_once_its_tar_was_served() { assert!( tmp.path() .join(".tar-integrity/packages") - .join(encode_segment("left-pad")) + .join(std::path::PathBuf::from_iter(package_segments("left-pad"))) .join("document.json") .is_file(), "the tar integrity lives in the shared hosted storage", diff --git a/pnpr/crates/pnpr/tests/server/tarball_proxy.rs b/pnpr/crates/pnpr/tests/server/tarball_proxy.rs index 6af10c040d..3ec4417244 100644 --- a/pnpr/crates/pnpr/tests/server/tarball_proxy.rs +++ b/pnpr/crates/pnpr/tests/server/tarball_proxy.rs @@ -1,8 +1,8 @@ use super::{ AccessList, AuthState, Body, PackageRules, Request, ServiceExt, StatusCode, TempDir, Value, access_rule, body_bytes, body_json, config_for, enable_osv, foo_packument, header, json, - mock_packument_for_tarball, osv_database, public_cache_blobs, router, router_with_auth, - sha512_integrity, upstream_endpoint_config, + mock_packument_for_tarball, osv_database, public_cache_blob, public_cache_blobs, router, + router_with_auth, sha512_integrity, upstream_endpoint_config, }; use crate::tokens::issue_token; @@ -629,7 +629,7 @@ async fn osv_refuses_vulnerable_cached_tarball_under_noncanonical_name() { // The warm-up must have written the tarball to the proxy cache, so the // screened request below genuinely exercises the cache-hit path rather than // silently falling back to a miss that would be refused anyway. - let cached_tarball = public_cache_blobs(&cache_dir, "foo").join("foo-0.0.1.tgz"); + let cached_tarball = public_cache_blob(&cache_dir, "foo", "foo-0.0.1.tgz"); assert!( cached_tarball.is_file(), "warm-up must populate the proxy cache at {cached_tarball:?}", @@ -923,7 +923,7 @@ async fn scoped_tarball_filename_is_canonicalized_before_fetch_and_cache() { .unwrap(); assert_eq!(canonical.status(), StatusCode::OK); assert_eq!(body_bytes(canonical.into_body()).await, bytes); - assert!(public_cache_blobs(&storage, "@types/node").join("node-20.0.0.tgz").exists()); + assert!(public_cache_blob(&storage, "@types/node", "node-20.0.0.tgz").exists()); assert!(!public_cache_blobs(&storage, "@types/node").join("@types").exists()); packument_mock.assert_async().await; mock.assert_async().await; diff --git a/pnpr/crates/pnpr/tests/server/tarball_verification.rs b/pnpr/crates/pnpr/tests/server/tarball_verification.rs index df016c1f7c..f6c29b0e1e 100644 --- a/pnpr/crates/pnpr/tests/server/tarball_verification.rs +++ b/pnpr/crates/pnpr/tests/server/tarball_verification.rs @@ -1,7 +1,7 @@ use super::{ Body, Duration, Request, ServiceExt, StatusCode, TempDir, Value, body_bytes, config_for, - foo_packument, json, mock_packument_for_tarball, public_cache_blobs, public_cache_pkg, router, - sha1_hex_of, sha512_integrity, tarball_cache_entries, to_bytes, + foo_packument, json, mock_packument_for_tarball, public_cache_blob, public_cache_blobs, + public_cache_pkg, router, sha1_hex_of, sha512_integrity, tarball_cache_entries, to_bytes, }; #[tokio::test] @@ -104,7 +104,7 @@ async fn tarball_route_preserves_basename_and_binds_to_declaring_version() { assert_eq!(body_bytes(first.into_body()).await, v1_bytes); let blobs_dir = public_cache_blobs(&storage, "foo"); - assert_eq!(tarball_cache_entries(&blobs_dir), vec!["foo-2.0.0.tgz".to_string()]); + assert_eq!(tarball_cache_entries(&blobs_dir), vec!["2.0.0.tgz".to_string()]); // A fresh instance over the same storage and the same origin replays the // cached entry; the `expect(1)` mocks prove the upstream is never @@ -168,7 +168,7 @@ async fn tampered_upstream_tarball_aborts_the_stream_and_is_never_cached() { let tmp = TempDir::new().unwrap(); let storage = tmp.path().to_path_buf(); - let cache_path = public_cache_blobs(&storage, "poisoned").join("poisoned-1.0.0.tgz"); + let cache_path = public_cache_blob(&storage, "poisoned", "poisoned-1.0.0.tgz"); let app = router(config_for(&upstream.url(), storage.clone())); let packument_response = app @@ -278,7 +278,7 @@ async fn missing_integrity_is_computed_cached_and_returned_in_the_packument() { assert_eq!(body_bytes(response.into_body()).await, bytes); assert_eq!( tarball_cache_entries(&public_cache_blobs(&storage, "foo")), - vec!["foo-1.0.0.tgz".to_string()], + vec!["1.0.0.tgz".to_string()], ); packument_mock.assert_async().await; tarball_mock.assert_async().await; @@ -1113,9 +1113,9 @@ async fn tarball_verification_finalizes_cache_with_no_tmp_leftover() { // built, leaving only the canonical cache path. let blobs_dir = public_cache_blobs(&cache_dir, "big"); let entries = tarball_cache_entries(&blobs_dir); - assert_eq!(entries, vec!["big-1.0.0.tgz".to_string()]); + assert_eq!(entries, vec!["1.0.0.tgz".to_string()]); - let cached = std::fs::read(blobs_dir.join("big-1.0.0.tgz")).unwrap(); + let cached = std::fs::read(blobs_dir.join("1.0.0.tgz")).unwrap(); assert_eq!(cached.len(), bytes.len()); assert_eq!(cached, bytes); } diff --git a/pnpr/crates/storage/src/layout.rs b/pnpr/crates/storage/src/layout.rs index 6133636dc5..f51fd1d715 100644 --- a/pnpr/crates/storage/src/layout.rs +++ b/pnpr/crates/storage/src/layout.rs @@ -7,14 +7,15 @@ //! orgs// one namespace per org //! ./ records every namespace shares //! / -//! packages//document.json -//! packages//blobs/ +//! packages///document.json +//! packages///blobs/ //! ./ records of this namespace //! ``` //! -//! Every name and file name is one [`encode_segment`]ed path segment, so a -//! name can never sit where an org, a namespace, a blob, or another name's -//! child does, and an object key is byte for byte the local path. +//! A package is the two segments of [`package_segments`] and a blob the one +//! segment of [`blob_segment`], so a name can never sit where an org, a +//! namespace, a blob, or another name's child does, and an object key is byte +//! for byte the local path. use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -27,6 +28,15 @@ pub(crate) const DOCUMENT_FILE: &str = "document.json"; pub(crate) const BLOBS_DIR: &str = "blobs"; const DEFAULT_NAMESPACE: &str = "default"; const ORGS_DIR: &str = "orgs"; +/// Opens the scope segment of a package, and the segment of a +/// [`VERSIONED_EXTENSIONS`] file not named after its package. +/// [`encode_segment`] never writes it. +const MARKER: char = '@'; +/// The files named `-`, which [`blob_segment`] +/// stores without the `-`. Adding an extension changes how a stored +/// file with it decodes, so one joins only with the ecosystem that introduces +/// it, and only when no other ecosystem stores files with it. +const VERSIONED_EXTENSIONS: [&str; 3] = [".tgz", ".sigstore", ".crate"]; const ESCAPE: u8 = b'!'; const HEX: &[u8; 16] = b"0123456789abcdef"; @@ -53,7 +63,7 @@ pub fn local_package_dir(root: &Path, org: &str, name: &str) -> PathBuf { let mut dir = root.to_path_buf(); dir.extend(namespace_segments(org)); dir.push(PACKAGES_DIR); - dir.push(encode_segment(name)); + dir.extend(package_segments(name)); dir } @@ -68,7 +78,7 @@ pub fn local_document_path(root: &Path, org: &str, name: &str) -> PathBuf { pub fn local_blob_path(root: &Path, org: &str, name: &str, file: &str) -> PathBuf { local_package_dir(root, org, name) .join(BLOBS_DIR) - .join(encode_segment(file)) + .join(blob_segment(name, file)) } /// Mark the local hosted store at `root` as being in [`LAYOUT_VERSION`]. @@ -81,6 +91,84 @@ pub fn write_local_layout_marker(root: &Path) -> std::io::Result<()> { std::fs::write(root.join(LAYOUT_FILE), marker) } +/// The directories of package `name` below `packages/`: `@` and the +/// rest of a scoped name, or `@` and a name without a scope. +#[must_use] +pub fn package_segments(name: &str) -> [String; 2] { + match name + .strip_prefix(MARKER) + .and_then(|rest| rest.split_once('/')) + { + Some((scope, rest)) if !scope.is_empty() => { + [format!("{MARKER}{}", encode_segment(scope)), encode_segment(rest)] + } + _ => [MARKER.to_string(), encode_segment(name)], + } +} + +/// The inverse of [`package_segments`], or `None` for segments it cannot have +/// produced. +#[must_use] +pub fn decode_package_segments(scope: &str, rest: &str) -> Option { + let decoded = decode_segment(rest)?; + let name = match scope.strip_prefix(MARKER)? { + "" => decoded, + encoded => format!("{MARKER}{}/{decoded}", decode_segment(encoded)?), + }; + (package_segments(&name) == [scope, rest]).then_some(name) +} + +/// The segment of package `name`'s blob `file` below `blobs/`. A `.tgz`, +/// `.sigstore` or `.crate` file drops the `-` it starts with, so a +/// tarball's segment is its version and extension, or keeps all of its name +/// behind `@` when it does not start with one. Any other file, such as an +/// image digest or a wheel, is its name. +#[must_use] +pub fn blob_segment(name: &str, file: &str) -> String { + if !is_versioned(file) { + return encode_segment(file); + } + match strip_basename(name, file) { + Some(rest) => encode_segment(rest), + None => format!("{MARKER}{}", encode_segment(file)), + } +} + +/// The inverse of [`blob_segment`], or `None` for a segment it cannot have +/// produced. +#[must_use] +pub fn decode_blob_segment(name: &str, segment: &str) -> Option { + let file = match segment.strip_prefix(MARKER) { + Some(file) => decode_segment(file)?, + None if is_versioned(segment) => { + format!("{}-{}", basename(name), decode_segment(segment)?) + } + None => decode_segment(segment)?, + }; + (blob_segment(name, &file) == segment).then_some(file) +} + +/// The part of `file` after `name`'s `-`, unless it is empty or +/// starts with `.`, which would make it a dot segment or hidden file. +fn strip_basename<'file>(name: &str, file: &'file str) -> Option<&'file str> { + let rest = file + .strip_prefix(basename(name))? + .strip_prefix('-')?; + (!rest.is_empty() && !rest.starts_with('.')).then_some(rest) +} + +fn is_versioned(file: &str) -> bool { + VERSIONED_EXTENSIONS + .iter() + .any(|extension| file.ends_with(extension)) +} + +/// The last `/`-separated component of `name`, as +/// [`pnpr_package_name::CanonicalPackageName`] names it. +fn basename(name: &str) -> &str { + name.rsplit_once('/').map_or(name, |(_, last)| last) +} + /// `text` as one path segment that is the same on a case-insensitive /// filesystem and in an object key: `[a-z0-9._-]` as is, every other byte as /// `!` and two lowercase hex digits. diff --git a/pnpr/crates/storage/src/layout/tests.rs b/pnpr/crates/storage/src/layout/tests.rs index 646f5cee98..34a5937e8f 100644 --- a/pnpr/crates/storage/src/layout/tests.rs +++ b/pnpr/crates/storage/src/layout/tests.rs @@ -1,4 +1,7 @@ -use super::{decode_segment, encode_segment, namespace_segments}; +use super::{ + blob_segment, decode_blob_segment, decode_package_segments, decode_segment, encode_segment, + namespace_segments, package_segments, +}; #[test] fn every_name_becomes_one_segment_that_decodes_back() { @@ -34,3 +37,64 @@ fn the_default_namespace_cannot_be_an_org() { assert_eq!(namespace_segments("default"), ["orgs", "default"]); assert_eq!(namespace_segments("npm~internal"), ["orgs", "npm!7einternal"]); } + +#[test] +fn every_package_is_two_segments_that_decode_back() { + for (name, segments) in [ + ("left-pad", ["@", "left-pad"]), + ("@pnpm-testing/fresh", ["@pnpm-testing", "fresh"]), + ("@Scope/App", ["@!53cope", "!41pp"]), + ("library/alpine", ["@", "library!2falpine"]), + ("@/empty-scope", ["@", "!40!2fempty-scope"]), + ("@no-slash", ["@", "!40no-slash"]), + ] { + assert_eq!(package_segments(name), segments, "{name}"); + assert_eq!(decode_package_segments(segments[0], segments[1]).as_deref(), Some(name)); + } +} + +#[test] +fn package_segments_the_layout_cannot_produce_do_not_decode() { + for (scope, rest) in [("scope", "pkg"), ("@a!2fb", "c"), ("@", "Upper"), ("@!zz", "pkg")] { + assert_eq!(decode_package_segments(scope, rest), None, "{scope}/{rest}"); + } +} + +#[test] +fn a_blob_named_after_its_package_drops_the_name() { + for (name, file, segment) in [ + ("left-pad", "left-pad-1.0.0.tgz", "1.0.0.tgz"), + ("@scope/pkg", "pkg-1.0.0-RC.1.sigstore", "1.0.0-!52!43.1.sigstore"), + ("serde", "serde-1.0.0.crate", "1.0.0.crate"), + ("library/alpine", "sha256-aa", "sha256-aa"), + ("my-pkg", "my_pkg-1.0-py3-none-any.whl", "my_pkg-1.0-py3-none-any.whl"), + ("demo", "demo-1.0.tar.gz", "demo-1.0.tar.gz"), + ("foo", "foo-.tgz", "@foo-.tgz"), + ("foo", "foo-..tgz", "@foo-..tgz"), + ("foo", "foobar-1.0.0.tgz", "@foobar-1.0.0.tgz"), + ("foo", "1.0.0.tgz", "@1.0.0.tgz"), + ] { + assert_eq!(blob_segment(name, file), segment, "{file}"); + assert_eq!(decode_blob_segment(name, segment).as_deref(), Some(file), "{segment}"); + } +} + +#[test] +fn blob_segments_the_layout_cannot_produce_do_not_decode() { + for segment in ["@foo-1.0.0.tgz", "@sha256-aa", ".tgz", "Upper"] { + assert_eq!(decode_blob_segment("foo", segment), None, "{segment:?}"); + } +} + +#[test] +fn files_of_ecosystems_without_versioned_names_keep_their_names() { + for file in [ + "sha256-aa", + "demo-1.0-py3-none-any.whl", + "demo-1.0-py3-none-any.whl.metadata", + "demo-1.0.tar.gz", + "demo-1.0.zip", + ] { + assert_eq!(blob_segment("demo", file), encode_segment(file), "{file}"); + } +} diff --git a/pnpr/crates/storage/src/lib.rs b/pnpr/crates/storage/src/lib.rs index 5544c3433d..8319768111 100644 --- a/pnpr/crates/storage/src/lib.rs +++ b/pnpr/crates/storage/src/lib.rs @@ -79,7 +79,10 @@ use tokio::{ io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; -use layout::{BLOBS_DIR, DOCUMENT_FILE, PACKAGES_DIR, decode_segment, encode_segment}; +use layout::{ + BLOBS_DIR, DOCUMENT_FILE, PACKAGES_DIR, blob_segment, decode_blob_segment, + decode_package_segments, package_segments, +}; pub const DOCUMENT_WRITE_RETRIES: usize = 8; /// Retries the commit path allows the document write: higher than the /// request-path budget because a sealed transaction has to converge, and a diff --git a/pnpr/crates/storage/src/local_store.rs b/pnpr/crates/storage/src/local_store.rs index 9377a95ff6..4b134d6586 100644 --- a/pnpr/crates/storage/src/local_store.rs +++ b/pnpr/crates/storage/src/local_store.rs @@ -5,8 +5,9 @@ use super::{ CachedDocument, CanonicalPackageName, DOCUMENT_FILE, DocumentStamp, DocumentWrite, Duration, ErrorKind, GetRange, HostedBackend, HostedBlobFile, HostedDocumentForUpdate, HostedDocumentVersion, PACKAGES_DIR, Path, PathBuf, RangedBlob, RegistryError, Result, - SeekFrom, StreamExt, SystemTime, async_trait, create_tmp_file, decode_segment, encode_segment, - fs, stream, streaming, unique_tmp_path, write_atomic, write_atomic_new, + SeekFrom, StreamExt, SystemTime, async_trait, blob_segment, create_tmp_file, + decode_blob_segment, decode_package_segments, fs, package_segments, stream, streaming, + unique_tmp_path, write_atomic, write_atomic_new, }; /// One on-disk store rooted at a single directory, laid out as @@ -203,17 +204,8 @@ impl Store { /// Every package with a document, decoded and sorted. pub(super) async fn list_package_names(&self) -> Result> { let mut names = Vec::new(); - let Some(mut entries) = read_dir_if_present(&self.root.join(PACKAGES_DIR)).await? else { - return Ok(names); - }; - while let Some(entry) = entries.next_entry().await? { - let segment = entry.file_name(); - let Some(name) = segment.to_str().and_then(decode_segment) else { - continue; - }; - if entry.file_type().await?.is_dir() - && is_regular_file(&entry.path().join(DOCUMENT_FILE)).await? - { + for (name, dir) in package_dirs(&self.root.join(PACKAGES_DIR)).await? { + if is_regular_file(&dir.join(DOCUMENT_FILE)).await? { names.push(name); } } @@ -222,9 +214,9 @@ impl Store { } pub(super) fn package_dir(&self, name: &CanonicalPackageName) -> PathBuf { - self.root - .join(PACKAGES_DIR) - .join(encode_segment(name.as_str())) + let mut dir = self.root.join(PACKAGES_DIR); + dir.extend(package_segments(name.as_str())); + dir } pub(super) fn document_path(&self, name: &CanonicalPackageName) -> PathBuf { @@ -234,7 +226,7 @@ impl Store { pub(super) fn blob_path(&self, name: &CanonicalPackageName, filename: &str) -> PathBuf { self.package_dir(name) .join(BLOBS_DIR) - .join(encode_segment(filename)) + .join(blob_segment(name.as_str(), filename)) } /// A record's path. The key's `/` separators become path components, so a @@ -333,6 +325,37 @@ impl Store { } /// Whether `path` is a regular file, not following a symlink. +/// Every package directory below `packages`, with its decoded name. +/// `file_type` does not follow a symlink, so a link cannot lead a listing, or +/// the deletions that act on it, out of the store. +pub(super) async fn package_dirs(packages: &Path) -> Result> { + let mut dirs = Vec::new(); + for (scope, scope_dir) in subdirs(packages).await? { + for (rest, dir) in subdirs(&scope_dir).await? { + if let Some(name) = decode_package_segments(&scope, &rest) { + dirs.push((name, dir)); + } + } + } + Ok(dirs) +} + +/// The directories directly in `dir` with UTF-8 names. +async fn subdirs(dir: &Path) -> Result> { + let mut dirs = Vec::new(); + let Some(mut entries) = read_dir_if_present(dir).await? else { + return Ok(dirs); + }; + while let Some(entry) = entries.next_entry().await? { + if entry.file_type().await?.is_dir() + && let Ok(name) = entry.file_name().into_string() + { + dirs.push((name, entry.path())); + } + } + Ok(dirs) +} + pub(super) async fn is_regular_file(path: &Path) -> Result { Ok(symlink_metadata_if_present(path).await?.is_some_and(|metadata| metadata.is_file())) } diff --git a/pnpr/crates/storage/src/local_store/hosted_backend.rs b/pnpr/crates/storage/src/local_store/hosted_backend.rs index 1c424946d8..1b7e2768af 100644 --- a/pnpr/crates/storage/src/local_store/hosted_backend.rs +++ b/pnpr/crates/storage/src/local_store/hosted_backend.rs @@ -4,8 +4,8 @@ use super::{ Arc, AsyncReadExt, AsyncSeekExt, BLOBS_DIR, BlobFinalize, Body, BoxStream, CanonicalPackageName, DOCUMENT_FILE, DocumentWrite, ErrorKind, GetRange, HostedBackend, HostedBlobFile, HostedDocumentForUpdate, HostedDocumentVersion, PACKAGES_DIR, Path, PathBuf, - RangedBlob, Result, SeekFrom, Store, StreamExt, async_trait, decode_segment, fs, - read_dir_if_present, stream, streaming, + RangedBlob, Result, SeekFrom, Store, StreamExt, async_trait, decode_blob_segment, fs, + package_dirs, read_dir_if_present, stream, streaming, }; /// The single-node filesystem backend. It owns its directory tree @@ -130,7 +130,7 @@ impl HostedBackend for Store { .file_name() .to_str() .filter(|file| !crate::atomic_write::is_atomic_write_temp_name(file)) - .and_then(decode_segment) + .and_then(|file| decode_blob_segment(name.as_str(), file)) && metadata.is_file() { list.push(blob); @@ -206,29 +206,6 @@ impl HostedBackend for Store { } } -/// Every package directory below `packages`, with its decoded name. -async fn package_dirs(packages: &Path) -> Result> { - let mut dirs = Vec::new(); - let Some(mut entries) = read_dir_if_present(packages).await? else { - return Ok(dirs); - }; - while let Some(entry) = entries.next_entry().await? { - // `file_type` does not follow a symlink, so a link cannot lead the - // inventory, and the deletions that act on it, out of the store. - if !entry.file_type().await?.is_dir() { - continue; - } - if let Some(name) = entry - .file_name() - .to_str() - .and_then(decode_segment) - { - dirs.push((name, entry.path())); - } - } - Ok(dirs) -} - /// The document and blobs of the package `name` in `dir`. async fn package_files(name: String, dir: PathBuf) -> Result> { let mut files = Vec::new(); @@ -252,7 +229,7 @@ async fn package_files(name: String, dir: PathBuf) -> Result .file_name() .to_str() .filter(|file| !crate::atomic_write::is_atomic_write_temp_name(file)) - .and_then(decode_segment) + .and_then(|file| decode_blob_segment(&name, file)) && metadata.is_file() { files.push(blob_file(&name, Some(blob), &metadata)?); diff --git a/pnpr/crates/storage/src/s3.rs b/pnpr/crates/storage/src/s3.rs index 250949a3fc..b3cd4e0df2 100644 --- a/pnpr/crates/storage/src/s3.rs +++ b/pnpr/crates/storage/src/s3.rs @@ -19,7 +19,8 @@ mod records; use crate::{ BLOBS_DIR, BlobFinalize, DOCUMENT_FILE, DocumentWrite, HostedBackend, HostedDocumentForUpdate, - HostedDocumentVersion, PACKAGES_DIR, decode_segment, encode_segment, + HostedDocumentVersion, PACKAGES_DIR, blob_segment, decode_blob_segment, + decode_package_segments, package_segments, }; use async_trait::async_trait; use axum::body::Body; @@ -346,13 +347,14 @@ impl S3Store { Err(error) => return Some(Err(error.into())), }; let rest = meta.location.as_ref().strip_prefix(normalized_prefix.as_str())?; + let (scope, rest) = rest.split_once('/')?; let (package, file) = rest.split_once('/')?; - let name = decode_segment(package)?; + let name = decode_package_segments(scope, package)?; let file = if file == DOCUMENT_FILE { PackageFile::Document } else { let blob = file.strip_prefix(BLOBS_DIR)?.strip_prefix('/')?; - PackageFile::Blob(decode_segment(blob)?) + PackageFile::Blob(decode_blob_segment(&name, blob)?) }; Some(Ok((name, file, meta))) } @@ -371,7 +373,7 @@ impl S3Store { while let Some(result) = listing.next().await { let meta = result?; if let Some(rest) = meta.location.as_ref().strip_prefix(prefix.as_str()) - && let Some(decoded) = decode_segment(rest) + && let Some(decoded) = decode_blob_segment(name.as_str(), rest) { blobs.push(decoded); } @@ -392,7 +394,8 @@ impl S3Store { } fn package_prefix(&self, name: &CanonicalPackageName) -> String { - format!("{}{PACKAGES_DIR}/{}/", self.prefix, encode_segment(name.as_str())) + let [scope, rest] = package_segments(name.as_str()); + format!("{}{PACKAGES_DIR}/{scope}/{rest}/", self.prefix) } fn document_key(&self, name: &CanonicalPackageName) -> ObjectPath { @@ -403,7 +406,7 @@ impl S3Store { ObjectPath::from(format!( "{}{BLOBS_DIR}/{}", self.package_prefix(name), - encode_segment(filename), + blob_segment(name.as_str(), filename), )) } } diff --git a/pnpr/crates/storage/src/streaming/tests.rs b/pnpr/crates/storage/src/streaming/tests.rs index f40f4133fb..2cccc78f2b 100644 --- a/pnpr/crates/storage/src/streaming/tests.rs +++ b/pnpr/crates/storage/src/streaming/tests.rs @@ -106,7 +106,7 @@ fn blob_tmp_entries(dir: &Path) -> Vec { .file_name() .to_string_lossy() .into_owned(); - name.starts_with("foo-1.0.0.tgz.tmp.").then_some(name) + name.starts_with("1.0.0.tgz.tmp.").then_some(name) }) .collect::>() }) @@ -120,7 +120,7 @@ fn cached_blobs_dir(cache: &Path) -> std::path::PathBuf { cache .join("~public/test") .join(crate::layout::PACKAGES_DIR) - .join(crate::layout::encode_segment("foo")) + .join(std::path::PathBuf::from_iter(crate::layout::package_segments("foo"))) .join(crate::layout::BLOBS_DIR) } @@ -170,7 +170,7 @@ async fn cancelling_in_flight_response_body_removes_tmp_file() { // `BlobWrite` is dropped and removes the tmp file. drop(chunks); assert!(blob_tmp_entries(&blobs_dir).is_empty()); - assert!(!blobs_dir.join("foo-1.0.0.tgz").exists()); + assert!(!blobs_dir.join("1.0.0.tgz").exists()); release.notify_one(); server.await.unwrap(); @@ -198,7 +198,7 @@ async fn oversized_response_is_rejected_and_tmp_is_removed() { // The temp file the rejected writer held is removed (its `Drop`). let blobs_dir = cached_blobs_dir(&cache); assert!(blob_tmp_entries(&blobs_dir).is_empty()); - assert!(!blobs_dir.join("foo-1.0.0.tgz").exists()); + assert!(!blobs_dir.join("1.0.0.tgz").exists()); } #[tokio::test] @@ -225,9 +225,7 @@ async fn download_returns_the_sha512_of_the_staged_bytes() { assert_eq!(computed.to_string(), sha512_integrity(bytes), "expected: {expected:?}"); assert_eq!( - tokio::fs::read(cached_blobs_dir(&cache).join("foo-1.0.0.tgz")) - .await - .unwrap(), + tokio::fs::read(cached_blobs_dir(&cache).join("1.0.0.tgz")).await.unwrap(), bytes, ); } @@ -252,11 +250,11 @@ async fn a_sealed_download_is_not_in_the_cache_until_promoted() { .unwrap(); let blobs_dir = cached_blobs_dir(&cache); - assert!(!blobs_dir.join("foo-1.0.0.tgz").exists()); + assert!(!blobs_dir.join("1.0.0.tgz").exists()); assert_eq!(blob_tmp_entries(&blobs_dir).len(), 1); drop(sealed); assert!(blob_tmp_entries(&blobs_dir).is_empty()); - assert!(!blobs_dir.join("foo-1.0.0.tgz").exists()); + assert!(!blobs_dir.join("1.0.0.tgz").exists()); } #[tokio::test] @@ -280,7 +278,7 @@ async fn download_stages_nothing_when_the_expected_integrity_mismatches() { assert!(matches!(err, BlobStreamError::Integrity(_))); let blobs_dir = cached_blobs_dir(&cache); assert!(blob_tmp_entries(&blobs_dir).is_empty()); - assert!(!blobs_dir.join("foo-1.0.0.tgz").exists()); + assert!(!blobs_dir.join("1.0.0.tgz").exists()); } async fn throttled_response(url: String) -> pnpm_network::ThrottledResponse { diff --git a/pnpr/crates/storage/src/tests.rs b/pnpr/crates/storage/src/tests.rs index 0d3dbd9d74..b34e6eba04 100644 --- a/pnpr/crates/storage/src/tests.rs +++ b/pnpr/crates/storage/src/tests.rs @@ -146,12 +146,12 @@ async fn packages_documents_and_blobs_have_one_path_each() { .await .unwrap(); - let package = tmp.path().join("storage/default/packages/!40scope!2f!41pp"); + let package = tmp.path().join("storage/default/packages/@scope/!41pp"); assert!(package.join("document.json").is_file()); - assert!(package.join("blobs/!41pp-1.0.0.tgz").is_file()); + assert!(package.join("blobs/1.0.0.tgz").is_file()); assert!( tmp.path() - .join("storage/orgs/npm!7einternal/packages/!40scope!2f!41pp/document.json") + .join("storage/orgs/npm!7einternal/packages/@scope/!41pp/document.json") .is_file(), ); } @@ -451,7 +451,11 @@ async fn symlinks_in_the_packages_directory_are_not_followed() { .write_hosted_document_if_current(&pkg("real"), b"{}", None) .await .unwrap(); - let packages = tmp.path().join("storage/default/packages"); + let scopes = tmp.path().join("storage/default/packages"); + fs::create_dir_all(tmp.path().join("outside-scope/pkg")).await.unwrap(); + fs::write(tmp.path().join("outside-scope/pkg/document.json"), b"{}").await.unwrap(); + fs::symlink(tmp.path().join("outside-scope"), scopes.join("@linked")).await.unwrap(); + let packages = scopes.join("@"); fs::symlink(&outside, packages.join("linked")).await.unwrap(); fs::create_dir_all(packages.join("real/blobs")).await.unwrap(); fs::symlink(outside.join("blobs/sha256-aa"), packages.join("real/blobs/sha256-aa")) diff --git a/pnpr/docs/storage.md b/pnpr/docs/storage.md index 2b3a7c5103..5cee730498 100644 --- a/pnpr/docs/storage.md +++ b/pnpr/docs/storage.md @@ -21,10 +21,10 @@ The hosted store has the same layout on disk and in a bucket: ```text / layout.json marks the layout version - default/packages// registries without an org - orgs//packages// one namespace per org -/packages//document.json -/packages//blobs/ + default/ registries without an org + orgs// one namespace per org +/packages///document.json +/packages///blobs/ ``` A hosted registry declared under an @@ -32,9 +32,18 @@ A hosted registry declared under an `org` uses the org `~`, stored as `orgs/!7e/`, for example `orgs/npm!7einternal/`. -Every package name, org and file name is one path segment. Lowercase -letters, digits, `.`, `_` and `-` are kept, and every other byte is written -as `!` and two hex digits, so `@scope/pkg` is stored as `!40scope!2fpkg`. +A scoped package is stored under `@//` and an unscoped one +under `@//`, so `@scope/pkg` is in `packages/@scope/pkg/` and `left-pad` +in `packages/@/left-pad/`. A `.tgz`, `.sigstore` or `.crate` file is +stored without the package's unscoped name and the `-` after it, so +`left-pad-1.0.0.tgz` is `blobs/1.0.0.tgz`. Such a file that does not start +with the package's name keeps its whole name behind a leading `@`. Every +other file, such as a container image layer or a Python wheel, keeps its +name. + +In every org, scope, name and file name, lowercase letters, digits, `.`, +`_` and `-` are kept. Every other byte is written as `!` and two hex digits, +so `@Scope/App` is stored as `@!53cope/!41pp/`. pnpr refuses to start on a store written by a newer pnpr, or on one in the layout pnpr used before `layout.json` existed. Convert such a store with