From f41c306cc8bef5eeef95430320cfb0b6827f9bc0 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Fri, 18 Sep 2026 10:43:00 +0200 Subject: [PATCH] feat(config): name the tools pnpm downloads and where they come from (#15043) pnpm downloads a Node.js runtime, a Bun runtime, a Deno runtime, a Yarn release and, since this week, a Python interpreter. Two of those could be pointed at a mirror, through two settings that shared neither a name nor a shape: `node-mirror:` and `python.downloadUrl`. Bun and Deno had none at all. `tools` names them, keyed by the tool, with `mirror` saying where its builds come from: tools: node: mirror: https://mirror.example.com/node/download bun: mirror: https://mirror.example.com/bun python: mirror: https://mirror.example.com/python-build-standalone/releases The key is the tool itself, with no category above it. Every taxonomy this could have used breaks on its next entry: Node.js is a runtime, Yarn is not, a Python interpreter arguably is, a Rust toolchain would not be, and a cargo subcommand certainly is not. `tools` inherits no such split and needs no rename when the next one arrives. It is also the word both tools that manage many runtimes use, in proto's `[tools.*]` and mise's tool-namespaced settings. A mirror carries the project's own layout below it, so only the host above it differs. That is what lets one string serve tools whose trees are otherwise nothing alike, and why `tools.node.mirror` is the base its release channels hang off, exactly as nodejs.org lays them out. This is the tool an ecosystem runs on, not where its packages come from. `registry`, `python.indexUrl` and `cargo.indexUrl` keep answering that, which is why `tools.python` and the `python` section can mean different things without competing. `python.downloadUrl` has not been released, so it is gone rather than deprecated, and its pending changeset now names the setting that replaces it. `node-mirror:` has shipped and stays: it names one release channel where the base names them all, so it decides the channel it names and leaves the rest to the base. Deno and Yarn are left out. Both read the GitHub API for release metadata and take asset URLs out of the response, which a base URL cannot stand in for. pnpm/pnpm#15042 covers the URL rewriting that would reach them. `pnpm pack-app` also downloads the Node.js it embeds through `tools.node` now. It passed no mirror at all, under a comment saying pacquet had no such setting, which stopped being true when `nodeDownloadMirrors` landed. It reads only `tools.node`: that runtime runs as the builder and is kept in the shared pack-app cache, so a repository naming `node-mirror:` would be choosing the program that packs everyone's app. Related to pnpm/pnpm#14945 --- .changeset/install-a-python-interpreter.md | 2 +- .changeset/tools-mirror.md | 27 +++++ pnpm/crates/cli/src/cli_args/env.rs | 11 +- pnpm/crates/cli/src/cli_args/pack_app.rs | 33 +++-- .../crates/cli/src/cli_args/pack_app/build.rs | 43 ++++--- pnpm/crates/cli/tests/suite/python.rs | 97 +++++++++++++-- pnpm/crates/config/src/env_overlay.rs | 1 + pnpm/crates/config/src/known_settings.rs | 1 + pnpm/crates/config/src/lib.rs | 4 +- pnpm/crates/config/src/settings.rs | 37 +++++- pnpm/crates/config/src/workspace_settings.rs | 8 ++ pnpm/crates/config/src/workspace_yaml.rs | 25 +++- .../crates/config/src/workspace_yaml/apply.rs | 4 +- .../crates/config/src/workspace_yaml/reset.rs | 2 +- .../config/src/workspace_yaml/resolved.rs | 1 + .../config/src/workspace_yaml/sections.rs | 85 ++++++++++++- .../config/src/workspace_yaml/settings.rs | 9 +- .../tests/workspace_settings.rs | 114 +++++++++++++++++- .../src/workspace_yaml/workspace_scope.rs | 5 + .../install_package_from_registry/tests.rs | 1 + .../src/bun_resolver.rs | 15 ++- .../src/read_bun_assets.rs | 24 ++-- .../src/read_bun_assets/tests.rs | 20 ++- .../src/get_node_mirror.rs | 27 +++-- .../src/get_node_mirror/tests.rs | 73 ++++++++++- .../src/node_resolver.rs | 21 +++- .../package-manager/src/add/specifier.rs | 4 +- .../resolver_setup.rs | 12 +- .../package-manager/src/update/latest.rs | 9 +- .../src/interpreter/download.rs | 14 ++- .../src/standalone.rs | 9 +- 31 files changed, 635 insertions(+), 103 deletions(-) create mode 100644 .changeset/tools-mirror.md diff --git a/.changeset/install-a-python-interpreter.md b/.changeset/install-a-python-interpreter.md index 891148fdea..1799f8799d 100644 --- a/.changeset/install-a-python-interpreter.md +++ b/.changeset/install-a-python-interpreter.md @@ -2,4 +2,4 @@ "pacquet": minor --- -`pnpm install` now installs a Python interpreter when no interpreter on the machine fits the project [#14945](https://github.com/pnpm/pnpm/issues/14945). The builds are [python-build-standalone](https://github.com/astral-sh/python-build-standalone)'s, which uv and rye install too. One interpreter is shared by every project on the machine, and a later install uses it without downloading anything. `runtimeOnFail` decides what an install with no interpreter that fits does, the way it does for a Node.js runtime. `error` reports the project instead of installing one. `warn` and `ignore` install with an interpreter the machine has that the project's `requires-python` rejects. `python.downloadUrl` names a mirror. +`pnpm install` now installs a Python interpreter when no interpreter on the machine fits the project [#14945](https://github.com/pnpm/pnpm/issues/14945). The builds are [python-build-standalone](https://github.com/astral-sh/python-build-standalone)'s, which uv and rye install too. One interpreter is shared by every project on the machine, and a later install uses it without downloading anything. `runtimeOnFail` decides what an install with no interpreter that fits does, the way it does for a Node.js runtime. `error` reports the project instead of installing one. `warn` and `ignore` install with an interpreter the machine has that the project's `requires-python` rejects. `tools.python.mirror` names a mirror. diff --git a/.changeset/tools-mirror.md b/.changeset/tools-mirror.md new file mode 100644 index 0000000000..c3a0b61bc2 --- /dev/null +++ b/.changeset/tools-mirror.md @@ -0,0 +1,27 @@ +--- +"pacquet": minor +--- + +`tools` names the programs pnpm downloads, and `mirror` says where each one comes from. + +```yaml +tools: + node: + mirror: https://mirror.example.com/node/download + channels: + nightly: https://nightly.example.com/ + bun: + mirror: https://mirror.example.com/bun + python: + mirror: https://mirror.example.com/python-build-standalone/releases +``` + +`node`, `bun` and `python` can be named. Any other tool is refused. + +`mirror` is the base a tool's own layout hangs off. + +`channels` sends one release channel elsewhere. A channel neither it nor `node-mirror:` names is left to `mirror`. Only `node` publishes channels, so naming them for another tool is refused. + +Set it in the global `config.yaml` or in `PNPM_CONFIG_TOOLS`. A `pnpm-workspace.yaml` that names a tool mirror is ignored. + +`pnpm pack-app` downloads the Node.js it embeds through `tools.node`. `node-mirror:` keeps working and names the same thing as an entry under `channels`. diff --git a/pnpm/crates/cli/src/cli_args/env.rs b/pnpm/crates/cli/src/cli_args/env.rs index 0575cdb0e2..677e53cfb9 100644 --- a/pnpm/crates/cli/src/cli_args/env.rs +++ b/pnpm/crates/cli/src/cli_args/env.rs @@ -5,7 +5,7 @@ use super::{global::handle_global_add, registry_client::build_registry_client}; use clap::Args; use derive_more::{Display, Error}; use miette::Diagnostic; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_runtime_node_resolver::{ get_node_mirror, parse_node_specifier, resolve_node_versions_with_auth, }; @@ -145,8 +145,13 @@ impl EnvArgs { pub async fn run_list(version_spec: Option, config: &Config) -> miette::Result { let specifier = parse_node_specifier(version_spec.as_deref().unwrap_or_default()) .map_err(miette::Report::new)?; - let mirror = - get_node_mirror(Some(&config.node_download_mirrors), &specifier.release_channel); + let channels = config.tool_channel_mirrors(Tool::Node); + let mirror = get_node_mirror( + config.tool_mirror(Tool::Node), + channels.get(&specifier.release_channel).map(String::as_str), + Some(&config.node_download_mirrors), + &specifier.release_channel, + ); let http_client = build_registry_client(config)?; let mut versions = resolve_node_versions_with_auth( &http_client, diff --git a/pnpm/crates/cli/src/cli_args/pack_app.rs b/pnpm/crates/cli/src/cli_args/pack_app.rs index fd58d52de9..b215cdab4a 100644 --- a/pnpm/crates/cli/src/cli_args/pack_app.rs +++ b/pnpm/crates/cli/src/cli_args/pack_app.rs @@ -19,9 +19,9 @@ //! home. use build::{ - SeaBuild, ad_hoc_sign_mac_binary, ensure_node_runtime, pnpm_home_dir, print_built, - reject_non_regular_output_file, reject_non_regular_outputs, resolve_builder_binary, - resolve_version, run_command, + EmbeddedRuntime, SeaBuild, ad_hoc_sign_mac_binary, ensure_node_runtime, pnpm_home_dir, + print_built, reject_non_regular_output_file, reject_non_regular_outputs, + resolve_builder_binary, resolve_version, run_command, }; use clap::Args; use config::{ @@ -302,14 +302,16 @@ impl PackAppArgs { // the serialized format has changed across Node.js minor releases, // so a blob produced by a builder of a different version than the // embedded runtime fails deserialization at startup. - let target_version = resolve_version(config, &requested_node_spec).await?; + let runtime = EmbeddedRuntime { + build_root, + version: resolve_version(config, &requested_node_spec).await?, + }; let build = SeaBuild { - builder_bin: resolve_builder_binary(&build_root, &target_version)?, + builder_bin: resolve_builder_binary(&runtime)?, pacquet_bin: std::env::current_exe() .into_diagnostic() .wrap_err("resolving the pnpm executable path")?, - build_root, - target_version, + runtime, dir, output_dir, output_name, @@ -446,8 +448,7 @@ impl SeaBuild<'_> { fn build_target(&self, target: &ParsedTarget) -> miette::Result { let embedded_node_bin = ensure_node_runtime( &self.pacquet_bin, - &self.build_root, - &self.target_version, + &self.runtime, &target.platform, &target.arch, target.libc.as_deref(), @@ -478,14 +479,12 @@ impl SeaBuild<'_> { self.build_sea(&output_file, &embedded_node_bin)?; ad_hoc_sign_mac_binary(target, &output_file, self.dir)?; - Ok( - format!( - " {}: {} (Node.js {})", - target.raw, - output_file.display(), - self.target_version, - ), - ) + Ok(format!( + " {}: {} (Node.js {})", + target.raw, + output_file.display(), + self.runtime.version, + )) } fn build_sea(&self, output_file: &Path, embedded_node_bin: &Path) -> miette::Result<()> { let sea_config = serde_json::json!({ diff --git a/pnpm/crates/cli/src/cli_args/pack_app/build.rs b/pnpm/crates/cli/src/cli_args/pack_app/build.rs index cc0fa9728c..fd825ad7f5 100644 --- a/pnpm/crates/cli/src/cli_args/pack_app/build.rs +++ b/pnpm/crates/cli/src/cli_args/pack_app/build.rs @@ -10,12 +10,18 @@ pub(super) struct SeaBuild<'a> { pub(super) output_dir: PathBuf, pub(super) output_name: String, pub(super) entry: PathBuf, - pub(super) build_root: PathBuf, - pub(super) target_version: String, + pub(super) runtime: EmbeddedRuntime, pub(super) builder_bin: PathBuf, pub(super) pacquet_bin: PathBuf, } +/// The Node.js the executables embed: which build, and where the copies +/// of it are kept. +pub(super) struct EmbeddedRuntime { + pub(super) build_root: PathBuf, + pub(super) version: String, +} + /// Reject a pre-existing symlink (or any non-regular file) at any /// target's final output path before downloading anything: a repo /// could commit `dist-app//` as a symlink pointing @@ -49,13 +55,11 @@ pub(super) fn print_built(results: &[String]) { /// Unlike pnpm — which reuses its own running interpreter when it already /// matches — pacquet has no host Node.js, so it always downloads a /// host-arch Node.js of the target version. -pub(super) fn resolve_builder_binary( - build_root: &Path, - target_version: &str, -) -> miette::Result { +pub(super) fn resolve_builder_binary(runtime: &EmbeddedRuntime) -> miette::Result { + let target_version = &runtime.version; if !builder_version_can_build_sea(target_version) { return Err(PackAppError::RuntimeTooOld { - version: target_version.to_string(), + version: target_version.clone(), major: MIN_BUILDER_VERSION.0, minor: MIN_BUILDER_VERSION.1, } @@ -65,8 +69,7 @@ pub(super) fn resolve_builder_binary( std::env::current_exe().into_diagnostic().wrap_err("resolving the pnpm executable path")?; ensure_node_runtime( &pacquet_bin, - build_root, - target_version, + runtime, pnpm_detect_libc::host_platform(), pnpm_detect_libc::host_arch(), // Pin libc to the host's. Otherwise a caller that set @@ -106,12 +109,12 @@ fn builder_version_can_build_sea(version: &str) -> bool { /// directory. pub(super) fn ensure_node_runtime( pacquet_bin: &Path, - build_root: &Path, - version: &str, + runtime: &EmbeddedRuntime, platform: &str, arch: &str, libc: Option<&str>, ) -> miette::Result { + let EmbeddedRuntime { build_root, version } = runtime; // Linux variants always need a libc pin (glibc or musl) so variant // selection is deterministic and doesn't depend on the host's detected // libc or the user's supportedArchitectures.libc config. @@ -146,7 +149,7 @@ pub(super) fn ensure_node_runtime( if !binary_path.exists() { return Err(PackAppError::NodeBinaryMissing { path: binary_path.display().to_string(), - version: version.to_string(), + version: version.clone(), } .into()); } @@ -159,10 +162,18 @@ fn node_binary_path(node_dir: &Path, platform: &str) -> PathBuf { pub(super) async fn resolve_version(config: &Config, specifier: &str) -> miette::Result { let parsed = parse_node_specifier(specifier).map_err(miette::Report::new)?; - // pacquet has no `node-download-mirrors` config field yet, so the - // override map is always absent and the official nodejs.org tree is - // used. Matches pnpm's default when `nodeDownloadMirrors` is unset. - let mirror = get_node_mirror(None, &parsed.release_channel); + // `node-mirror:` is deliberately not read here. A workspace + // may name it, and this runtime is executed as the builder and kept + // in the shared pack-app cache, so a repository naming it would be + // choosing the program that packs everyone's app. `tools` carries no + // such risk, being the machine's own. + let channels = config.tool_channel_mirrors(pnpm_config::Tool::Node); + let mirror = get_node_mirror( + config.tool_mirror(pnpm_config::Tool::Node), + channels.get(&parsed.release_channel).map(String::as_str), + None, + &parsed.release_channel, + ); let http_client = build_http_client(config)?; let version = resolve_node_version(&http_client, &parsed.version_specifier, Some(&mirror)) .await diff --git a/pnpm/crates/cli/tests/suite/python.rs b/pnpm/crates/cli/tests/suite/python.rs index 7f75cea22d..4693ecc387 100644 --- a/pnpm/crates/cli/tests/suite/python.rs +++ b/pnpm/crates/cli/tests/suite/python.rs @@ -189,6 +189,78 @@ fn add_python_settings(root: &Path, settings: &str) { .unwrap(); } +/// A repository that named a mirror would be choosing which program runs +/// on the machine of everyone who clones it, and a release's checksums +/// come from the mirror that serves its files, so the download verifies +/// only that the mirror agrees with itself. +/// +/// The machine and the repository name mirrors serving different +/// releases, and only the repository's serves the pinned one. Honouring +/// the workspace would install it, so the install failing is what says +/// the workspace was not read. The machine's mirror is named in the +/// global `config.yaml` rather than the environment, because the +/// environment outranks the workspace either way and would prove +/// nothing. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn a_repository_cannot_name_a_mirror() { + use command_extra::CommandExtra as _; + + let root = tempfile::tempdir().unwrap(); + let mut machine = mockito::Server::new_async().await; + let mut repository = mockito::Server::new_async().await; + project(root.path(), "https://unused.invalid", &[]); + let machines = serve_interpreter(&mut machine, &["3.13.90"]).await; + let pinned = serve_interpreter(&mut repository, &["3.13.95"]).await; + + let config = root.path().join(".config/pnpm"); + fs::create_dir_all(&config).unwrap(); + fs::write( + config.join("config.yaml"), + format!("tools:\n python:\n mirror: '{}'\n", machine.url()), + ) + .unwrap(); + let path = root.path().join("pnpm-workspace.yaml"); + let mut workspace = fs::read_to_string(&path).unwrap(); + writeln!(workspace, "tools:\n python:\n mirror: '{}'", repository.url()).unwrap(); + fs::write(&path, workspace).unwrap(); + fs::write( + root.path().join("pyproject.toml"), + "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.95'\ndependencies = []\n", + ) + .unwrap(); + + assert_failure_contains( + pacquet_in(root.path()) + .with_env("XDG_CONFIG_HOME", root.path().join(".config")) + .arg("install"), + "3.13.95", + ); + for mock in pinned { + assert!(!mock.matched_async().await, "the repository's mirror was read"); + } + // Without this the test would also pass if the machine's mirror were + // never found either: the install would reach for the real releases, + // fail naming the same version, and leave the repository's untouched. + let mut read = false; + for mock in machines { + read |= mock.matched_async().await; + } + assert!(read, "the machine's mirror was not read"); +} + +/// A pnpm that downloads interpreters from `mirror`. +/// +/// A tool mirror is the machine's to name, so a test names one the way a +/// user does, through the environment, rather than through the workspace +/// file the run would ignore it in. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn pacquet_with_python_mirror(root: &Path, mirror: &str) -> Command { + let mut command = pacquet_in(root); + command.env("PNPM_CONFIG_TOOLS", format!(r#"{{"python":{{"mirror":"{mirror}"}}}}"#)); + command +} + fn add_supported_architectures(root: &Path, platforms: &[&str]) { let path = root.join("pnpm-workspace.yaml"); let mut workspace = fs::read_to_string(&path).unwrap(); @@ -554,14 +626,14 @@ async fn installs_an_interpreter_no_machine_has_and_reuses_it() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let release = serve_interpreter(&mut server, &["3.13.99"]).await; - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); + let mirror = server.url(); let install = || { fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.99'\ndependencies = []\n", ) .unwrap(); - pacquet_in(root.path()) + pacquet_with_python_mirror(root.path(), &mirror) }; install() @@ -598,15 +670,15 @@ async fn a_pin_the_release_moved_past_installs_the_version_it_has() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.96"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '>=3.13.90,<3.14'\ndependencies = []\n", ) .unwrap(); fs::write(root.path().join(".python-version"), "3.13.95\n").unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); - let output = pacquet_in(root.path()) + let output = pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .output() .unwrap(); @@ -626,15 +698,15 @@ async fn a_pin_the_project_refuses_installs_a_version_it_accepts() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.94", "3.13.93"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '>=3.13.94,<3.14'\ndependencies = []\n", ) .unwrap(); fs::write(root.path().join(".python-version"), "3.13.93\n").unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); - let output = pacquet_in(root.path()) + let output = pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .output() .unwrap(); @@ -667,14 +739,14 @@ async fn refuses_an_interpreter_the_release_does_not_name() { .with_body("not the interpreter the release names") .create_async() .await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.97'\ndependencies = []\n", ) .unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); assert_failure_contains( - pacquet_in(root.path()).arg("install"), + pacquet_with_python_mirror(root.path(), &mirror).arg("install"), "is not the one the release names", ); assert!( @@ -695,21 +767,22 @@ async fn an_interpreter_is_installed_only_where_the_install_may_download() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.98"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.98'\ndependencies = []\n", ) .unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); assert_failure_contains( - pacquet_in(root.path()).args(["install", "--runtime-on-fail=error"]), + pacquet_with_python_mirror(root.path(), &mirror) + .args(["install", "--runtime-on-fail=error"]), "no Python interpreter for", ); assert_failure_contains( - pacquet_in(root.path()).args(["install", "--offline"]), + pacquet_with_python_mirror(root.path(), &mirror).args(["install", "--offline"]), "no Python interpreter for", ); - pacquet_in(root.path()) + pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .assert() .success(); diff --git a/pnpm/crates/config/src/env_overlay.rs b/pnpm/crates/config/src/env_overlay.rs index 9d14b7ae15..db87aa3db6 100644 --- a/pnpm/crates/config/src/env_overlay.rs +++ b/pnpm/crates/config/src/env_overlay.rs @@ -363,6 +363,7 @@ impl WorkspaceSettings { ); json_field!(settings, Sys, changed_files_ignore_pattern, "CHANGED_FILES_IGNORE_PATTERN"); json_field!(settings, Sys, supported_architectures, "SUPPORTED_ARCHITECTURES"); + json_field!(settings, Sys, tools, "TOOLS"); json_field!(settings, Sys, ignored_optional_dependencies, "IGNORED_OPTIONAL_DEPENDENCIES"); json_field!(settings, Sys, overrides, "OVERRIDES"); json_field!(settings, Sys, package_extensions, "PACKAGE_EXTENSIONS"); diff --git a/pnpm/crates/config/src/known_settings.rs b/pnpm/crates/config/src/known_settings.rs index 9b8b2815a2..be198510b1 100644 --- a/pnpm/crates/config/src/known_settings.rs +++ b/pnpm/crates/config/src/known_settings.rs @@ -47,6 +47,7 @@ const TYPED_WORKSPACE_MANIFEST_KEYS: &[&str] = &[ "requiredScripts", "sideEffectsCache", "supportedArchitectures", + "tools", "update", "updateConfig", "versioning", diff --git a/pnpm/crates/config/src/lib.rs b/pnpm/crates/config/src/lib.rs index 044c283c8d..ff083e7ecc 100644 --- a/pnpm/crates/config/src/lib.rs +++ b/pnpm/crates/config/src/lib.rs @@ -38,8 +38,8 @@ pub use workspace_yaml::{ AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, GLOBAL_CONFIG_YAML_FILENAME, LoadWorkspaceYamlError, PackageExtension, PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, PythonSettings, RemoteSideEffectsCacheSettings, - TaskSettings, UpdateConfig, UpdateSettings, WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, - WorkspaceSettings, decided_allow_builds, + TaskSettings, Tool, ToolSettings, UpdateConfig, UpdateSettings, WORKSPACE_MANIFEST_FILENAME, + WorkspaceKeyIssues, WorkspaceSettings, decided_allow_builds, package_configs::{self, PackageConfigsSetting, ProjectConfig, ProjectConfigMultiMatch}, registries::{self, RegistryDeclaration, RegistryEntry, RegistryLookups}, workspace_root_or, diff --git a/pnpm/crates/config/src/settings.rs b/pnpm/crates/config/src/settings.rs index eaaf68b75d..d75d9a15f1 100644 --- a/pnpm/crates/config/src/settings.rs +++ b/pnpm/crates/config/src/settings.rs @@ -5,8 +5,8 @@ use super::{ PackageManagerBootstrap, PatchGroupRecord, PatchInput, Path, PathBuf, Pipe, PmOnFail, ProjectConfig, PythonSettings, RegistryOptions, RemoteSideEffectsCacheSettings, ResolutionMode, ResolvePatchedDependenciesError, RuntimeOnFail, SaveWorkspaceProtocol, ScriptsPrependNodePath, - SmartDefault, StoreDir, TrustPolicy, VerifyDepsBeforeRun, WorkspaceKeyIssues, - create_hex_hash_from_file, default_cache_dir, default_child_concurrency, + SmartDefault, StoreDir, Tool, ToolSettings, TrustPolicy, VerifyDepsBeforeRun, + WorkspaceKeyIssues, create_hex_hash_from_file, default_cache_dir, default_child_concurrency, default_enable_global_virtual_store, default_fetch_min_speed_ki_bps, default_fetch_retries, default_fetch_retry_factor, default_fetch_retry_maxtimeout, default_fetch_retry_mintimeout, default_fetch_timeout, default_fetch_warn_timeout_ms, default_git_shallow_hosts, @@ -971,6 +971,8 @@ pub struct Config { /// Cargo dependency management declared by the workspace. pub cargo: CargoSettings, pub python: PythonSettings, + /// `tools` from `pnpm-workspace.yaml`, keyed by tool name. + pub tools: BTreeMap, pub remote_side_effects_cache: Option, @@ -1695,6 +1697,37 @@ pub struct Config { } impl Config { + /// Where the builds of one tool are downloaded from, as + /// `tools..mirror` names it, without the trailing slash a + /// caller joins onto. + #[must_use] + pub fn tool_mirror(&self, tool: Tool) -> Option<&str> { + self.tools + .get(&tool)? + .mirror + .as_deref() + .map(|mirror| mirror.trim_end_matches('/')) + } + + /// Where each line of a tool's builds is downloaded from, as + /// `tools..channels` names them. A line it does not name is + /// left to [`Self::tool_mirror`]. + #[must_use] + pub fn tool_channel_mirrors(&self, tool: Tool) -> HashMap { + self.tools + .get(&tool) + .and_then(|tool| tool.channels.as_ref()) + .map(|channels| { + channels + .iter() + .map(|(channel, mirror)| { + (channel.clone(), mirror.trim_end_matches('/').to_string()) + }) + .collect() + }) + .unwrap_or_default() + } + #[must_use] pub fn new() -> Self { Self::default() diff --git a/pnpm/crates/config/src/workspace_settings.rs b/pnpm/crates/config/src/workspace_settings.rs index e20fffa3dd..031087bc74 100644 --- a/pnpm/crates/config/src/workspace_settings.rs +++ b/pnpm/crates/config/src/workspace_settings.rs @@ -99,6 +99,14 @@ impl Config { settings.scope = None; settings.global_dir = None; settings.global_bin_dir = None; + // A mirror decides where the binary pnpm runs an ecosystem with is + // downloaded from, and a release's checksums come from the mirror + // that serves its files, so verifying the download says only that + // the mirror agrees with itself. A repository that named one would + // be choosing which program runs on the machine of everyone who + // clones it. `tools` therefore comes from the global `config.yaml` + // and `PNPM_CONFIG_TOOLS` only. + settings.tools = None; // Noted rather than assigned, so an `enableGlobalVirtualStore` / // `virtualStoreDir` set in the global `config.yaml` still counts as // "explicitly set" when the workspace yaml leaves it unset. diff --git a/pnpm/crates/config/src/workspace_yaml.rs b/pnpm/crates/config/src/workspace_yaml.rs index 8041c84014..79e209cc42 100644 --- a/pnpm/crates/config/src/workspace_yaml.rs +++ b/pnpm/crates/config/src/workspace_yaml.rs @@ -1,11 +1,12 @@ pub mod package_configs; pub mod registries; pub use error::LoadWorkspaceYamlError; +pub(crate) use sections::deserialize_tools; pub use sections::{ AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, PackageExtension, PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, PythonSettings, RemoteSideEffectsCacheSettings, SideEffectsCacheSetting, SideEffectsCacheSettings, - TaskSettings, UpdateConfig, UpdateSettings, decided_allow_builds, + TaskSettings, Tool, ToolSettings, UpdateConfig, UpdateSettings, decided_allow_builds, }; pub use settings::WorkspaceSettings; @@ -75,6 +76,28 @@ fn overlay(target: &mut Setting, value: Option) { } } +/// [`overlay`] for the tools, which every layer answers for separately. +/// +/// The tools are independent of one another, so a workspace naming a Bun +/// mirror has said nothing about Node.js and must not drop the one the +/// machine's own config names. The same holds a level down: a layer that +/// names a tool's release channels has not said where the rest of its +/// builds come from. +fn overlay_tools( + target: &mut BTreeMap, + value: Option>, +) { + for (tool, named) in value.into_iter().flatten() { + let settings = target.entry(tool).or_default(); + overlay_some(&mut settings.mirror, named.mirror); + match (&mut settings.channels, named.channels) { + (Some(channels), Some(named)) => channels.extend(named), + (channels @ None, named @ Some(_)) => *channels = named, + (_, None) => {} + } + } +} + /// [`overlay`] for a target that is itself optional: an unset field leaves /// whatever the previous layer recorded, including its absence. fn overlay_some(target: &mut Option, value: Option) { diff --git a/pnpm/crates/config/src/workspace_yaml/apply.rs b/pnpm/crates/config/src/workspace_yaml/apply.rs index 8da5a9072c..f36ea5c608 100644 --- a/pnpm/crates/config/src/workspace_yaml/apply.rs +++ b/pnpm/crates/config/src/workspace_yaml/apply.rs @@ -1,7 +1,8 @@ use super::{ Config, Path, PnpmfileSetting, ProxyKeys, ProxyValue, SideEffectsCacheSetting, StoreDir, UpdateConfig, WorkspaceSettings, decided_allow_builds, no_proxy_scalar, normalize_registry_url, - overlay, overlay_some, registries, resolve, resolve_child_concurrency, warn_deprecated_pairing, + overlay, overlay_some, overlay_tools, registries, resolve, resolve_child_concurrency, + warn_deprecated_pairing, }; impl WorkspaceSettings { @@ -164,6 +165,7 @@ impl WorkspaceSettings { overlay_some(&mut config.pnpr_server, self.pnpr_server.take()); overlay(&mut config.cargo, self.cargo.take()); overlay(&mut config.python, self.python.take()); + overlay_tools(&mut config.tools, self.tools.take()); if let Some(v) = self.remote_side_effects_cache.take() { config.remote_side_effects_cache.get_or_insert_default().overlay(v); } diff --git a/pnpm/crates/config/src/workspace_yaml/reset.rs b/pnpm/crates/config/src/workspace_yaml/reset.rs index 35d05e0eb8..7c851bcd4b 100644 --- a/pnpm/crates/config/src/workspace_yaml/reset.rs +++ b/pnpm/crates/config/src/workspace_yaml/reset.rs @@ -122,7 +122,7 @@ impl WorkspaceSettings { global_pnpmfile, pnpmfile, global_dir, global_bin_dir, cache_dir, prefer_frozen_lockfile, lockfile, merge_git_branch_lockfiles, optimistic_repeat_install, minimum_release_age, global_shims, frozen_lockfile, - registry, scope, pnpr_server, cargo, python, remote_side_effects_cache, + registry, scope, pnpr_server, cargo, python, tools, remote_side_effects_cache, reporter_hide_prefix, max_sockets, patched_dependencies, patches_dir, config_dependencies, dangerously_allow_all_builds, strict_dep_builds, ignore_scripts, ignore_pnpmfile, git_checks, engine_strict, node_version, diff --git a/pnpm/crates/config/src/workspace_yaml/resolved.rs b/pnpm/crates/config/src/workspace_yaml/resolved.rs index a9ab69f043..f7de610fce 100644 --- a/pnpm/crates/config/src/workspace_yaml/resolved.rs +++ b/pnpm/crates/config/src/workspace_yaml/resolved.rs @@ -107,6 +107,7 @@ impl WorkspaceSettings { pnpr_server: config.pnpr_server.clone(), cargo: Some(config.cargo.clone()), python: Some(config.python.clone()), + tools: Some(config.tools.clone()), remote_side_effects_cache: config.remote_side_effects_cache.clone(), reporter_hide_prefix: config.reporter_hide_prefix, max_sockets: config.max_sockets, diff --git a/pnpm/crates/config/src/workspace_yaml/sections.rs b/pnpm/crates/config/src/workspace_yaml/sections.rs index a6840ff683..10c39195a4 100644 --- a/pnpm/crates/config/src/workspace_yaml/sections.rs +++ b/pnpm/crates/config/src/workspace_yaml/sections.rs @@ -99,6 +99,64 @@ pub struct RemoteSideEffectsCacheSettings { pub private_key: Option, } +/// A program pnpm downloads through a base URL, which is what a mirror +/// can replace. +/// +/// Deno and Yarn are absent on purpose: both read the GitHub API for +/// their release metadata and take asset URLs out of the response, so a +/// base URL cannot stand in for either. Naming one would promise +/// something pnpm cannot do, and a closed set says so where the +/// configuration is written rather than after it is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Tool { + Node, + Bun, + Python, +} + +/// What pnpm is told about one tool it downloads, keyed by the tool +/// under `tools`. +/// +/// A tool here is a program pnpm fetches to run something with: a +/// JavaScript runtime, a Python interpreter, another package manager. +/// Where the packages of an ecosystem come from is a separate question, +/// answered by `registry`, `python.indexUrl` and `cargo.indexUrl`. +/// +/// Read from the global `config.yaml` and `PNPM_CONFIG_TOOLS` only. A +/// `pnpm-workspace.yaml` that names one is ignored, because naming a +/// mirror chooses which program runs on the machine of everyone who +/// clones the repository. +#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default, deny_unknown_fields)] +pub struct ToolSettings { + /// Where this tool's builds are downloaded from, in place of the + /// project that publishes them, as the base URL its own layout hangs + /// off. A tool that publishes several lines of builds has the line + /// below this base, the way its own tree lays them out. + pub mirror: Option, + /// Where one line of this tool's builds comes from, for a tool that + /// publishes more than one and a line that does not come from the + /// same place as the rest. An entry here answers for the channel it + /// names; every other channel is left to [`Self::mirror`]. + /// + /// Only Node.js publishes channels of the tools pnpm downloads, so + /// naming them for another is refused rather than left to do + /// nothing. [`Tool::has_channels`] is what decides. + pub channels: Option>, +} + +impl Tool { + /// Whether this tool publishes more than one line of builds. + #[must_use] + pub fn has_channels(self) -> bool { + match self { + Self::Node => true, + Self::Bun | Self::Python => false, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] #[serde(rename_all = "camelCase", default, deny_unknown_fields)] pub struct CargoSettings { @@ -136,11 +194,6 @@ pub struct PythonSettings { /// version such as `3.12` or a full one such as `3.12.7`. Empty locks /// for the version of the interpreter the install runs on. pub python_versions: Vec, - /// Where the interpreters pnpm installs are downloaded from, as the - /// releases URL of a [python-build-standalone] mirror. - /// - /// [python-build-standalone]: https://github.com/astral-sh/python-build-standalone - pub download_url: String, } impl Default for PythonSettings { @@ -155,7 +208,6 @@ impl Default for PythonSettings { extras: Vec::new(), groups: vec!["dev".to_string()], python_versions: Vec::new(), - download_url: DEFAULT_PYTHON_DOWNLOAD_URL.to_string(), } } } @@ -485,3 +537,24 @@ pub struct PeerDependencyMeta { #[serde(skip_serializing_if = "Option::is_none")] pub optional: Option, } + +/// Refuse `channels` for a tool that publishes one line of builds, where +/// it would otherwise sit in the configuration doing nothing. +/// +/// Checked as the value is read so that every source is covered: the +/// workspace file, the global `config.yaml` and `PNPM_CONFIG_TOOLS` all +/// arrive through serde. +pub(crate) fn deserialize_tools<'de, Deser: Deserializer<'de>>( + deserializer: Deser, +) -> Result>, Deser::Error> { + use serde::de::Error as _; + let tools = Option::>::deserialize(deserializer)?; + for (tool, settings) in tools.iter().flatten() { + if settings.channels.is_some() && !tool.has_channels() { + return Err(Deser::Error::custom(format!( + "{tool:?} publishes one line of builds, so it has no channels to name", + ))); + } + } + Ok(tools) +} diff --git a/pnpm/crates/config/src/workspace_yaml/settings.rs b/pnpm/crates/config/src/workspace_yaml/settings.rs index 398c110e91..53ebbd3083 100644 --- a/pnpm/crates/config/src/workspace_yaml/settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/settings.rs @@ -7,8 +7,9 @@ use super::{ PeerDependencyRules, Pipe, PmOnFail, PnpmfileSetting, PythonSettings, RegistryEntry, RemoteSideEffectsCacheSettings, ResolutionMode, RuntimeOnFail, SCHEMA_DIRECTIVE_KEY, SaveWorkspaceProtocol, ScriptsPrependNodePath, SideEffectsCacheSetting, SupportedArchitectures, - TaskSettings, TrustPolicy, UpdateConfig, UpdateSettings, VerifyDepsBeforeRun, VirtualStoreType, - WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, fs, redact_and_sanitize, + TaskSettings, Tool, ToolSettings, TrustPolicy, UpdateConfig, UpdateSettings, + VerifyDepsBeforeRun, VirtualStoreType, WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, fs, + redact_and_sanitize, }; /// `serde` helper for fields that need to distinguish "missing key" @@ -165,6 +166,10 @@ pub struct WorkspaceSettings { pub pnpr_server: Option, pub cargo: Option, pub python: Option, + /// `tools` from `pnpm-workspace.yaml`: what pnpm is told about the + /// programs it downloads, keyed by tool name. See [`ToolSettings`]. + #[serde(default, deserialize_with = "crate::workspace_yaml::deserialize_tools")] + pub tools: Option>, pub remote_side_effects_cache: Option, pub https_proxy: Option, pub http_proxy: Option, diff --git a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs index 2fa5c06b60..ecfa36d4c5 100644 --- a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs @@ -312,9 +312,120 @@ cargo: assert!(settings.cargo.is_none()); } +#[test] +fn tool_settings_parse_and_apply() { + let yaml = "tools:\n node:\n mirror: https://mirror.example.test/node/download\n python:\n mirror: https://mirror.example.test/python-build-standalone/releases\n bun:\n mirror: https://mirror.example.test/bun\n"; + let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + let mut config = Config::default(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!( + config.tool_mirror(crate::Tool::Node), + Some("https://mirror.example.test/node/download"), + ); + assert_eq!( + config.tool_mirror(crate::Tool::Python), + Some("https://mirror.example.test/python-build-standalone/releases"), + ); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://mirror.example.test/bun")); + + // A mirror says what this machine can reach, which is the user's to + // say, so it survives the filter the global config is read through. + // The workspace layer drops it; `a_repository_cannot_name_a_mirror` + // covers that end. + let mut settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + settings.clear_workspace_only_fields(); + assert!(settings.tools.is_some()); + let unknown = + serde_saphyr::from_str::("tools:\n node:\n unknown: true\n"); + assert!(unknown.is_err()); +} + +/// Deno and Yarn are not reachable through a base URL at all, so the +/// setting cannot name one: the closed key set says so where the value +/// is written, rather than leaving an entry that does nothing. +#[test] +fn refuses_a_tool_pnpm_does_not_download() { + let error = serde_saphyr::from_str::( + "tools:\n deno:\n mirror: https://mirror.example.test/deno\n", + ) + .expect_err("deno is not a tool pnpm downloads through a mirror"); + assert!(format!("{error}").contains("deno"), "{error}"); + + let known = serde_saphyr::from_str::( + "tools:\n node:\n mirror: https://mirror.example.test/node\n", + ); + assert!(known.is_ok()); +} + +/// Only Node.js publishes more than one line of builds, so naming +/// channels for another would sit in the file doing nothing. +#[test] +fn refuses_channels_for_a_tool_that_has_none() { + let error = serde_saphyr::from_str::( + "tools:\n bun:\n channels:\n canary: https://mirror.example.test/bun\n", + ) + .expect_err("bun publishes one line of builds"); + assert!(format!("{error}").contains("channels"), "{error}"); + + let node = serde_saphyr::from_str::( + "tools:\n node:\n channels:\n nightly: https://mirror.example.test/node\n", + ); + assert!(node.is_ok()); +} + +/// Each tool is answered for separately, so naming one leaves the +/// mirrors the machine's own config names for the others alone. +#[test] +fn a_workspace_tool_keeps_the_mirrors_named_elsewhere() { + let mut config = Config::default(); + let global: WorkspaceSettings = + serde_saphyr::from_str("tools:\n node:\n mirror: https://global.example.test/node\n") + .unwrap(); + global.apply_to(&mut config, Path::new("/workspace")); + let workspace: WorkspaceSettings = + serde_saphyr::from_str("tools:\n bun:\n mirror: https://workspace.example.test/bun\n") + .unwrap(); + workspace.apply_to(&mut config, Path::new("/workspace")); + + assert_eq!(config.tool_mirror(crate::Tool::Node), Some("https://global.example.test/node")); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://workspace.example.test/bun")); +} + +#[test] +fn a_tool_channel_is_read_beside_the_base_it_refines() { + let yaml = "tools:\n node:\n mirror: https://mirror.example.test/node/download\n channels:\n nightly: https://nightly.example.test/\n"; + let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + let mut config = Config::default(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!( + config.tool_mirror(crate::Tool::Node), + Some("https://mirror.example.test/node/download"), + ); + assert_eq!( + config + .tool_channel_mirrors(crate::Tool::Node) + .get("nightly") + .map(String::as_str), + Some("https://nightly.example.test"), + ); + assert!(!config.tool_channel_mirrors(crate::Tool::Node).contains_key("release")); + assert!(config.tool_channel_mirrors(crate::Tool::Bun).is_empty()); +} + +/// A caller joins a path onto what it is given. +#[test] +fn a_tool_mirror_is_read_without_its_trailing_slash() { + let mut config = Config::default(); + let settings: WorkspaceSettings = + serde_saphyr::from_str("tools:\n bun:\n mirror: https://mirror.example.test/bun/\n") + .unwrap(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://mirror.example.test/bun")); +} + #[test] fn python_settings_parse_apply_and_remain_workspace_only() { - let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n overrides: [demo>=2]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n pythonVersions: ['3.12', '3.13']\n downloadUrl: https://mirror.example.test/releases\n"; + let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n overrides: [demo>=2]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n pythonVersions: ['3.12', '3.13']\n"; let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); let mut config = Config::default(); settings.apply_to(&mut config, Path::new("/workspace")); @@ -327,7 +438,6 @@ fn python_settings_parse_apply_and_remain_workspace_only() { assert_eq!(config.python.extras, ["speed"]); assert_eq!(config.python.groups, ["test"]); assert_eq!(config.python.python_versions, ["3.12", "3.13"]); - assert_eq!(config.python.download_url, "https://mirror.example.test/releases"); let mut settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); settings.clear_workspace_only_fields(); assert!(settings.python.is_none()); diff --git a/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs b/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs index 80fa76a237..6f65eaa593 100644 --- a/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs +++ b/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs @@ -52,6 +52,11 @@ impl WorkspaceSettings { self.clear_workspace_hooks_fields(); } + /// `tools` is deliberately absent: a mirror answers "what can this + /// machine reach", which is the user's to say and belongs in their + /// global config, the same way `nodeDownloadMirrors` does. Clearing + /// it here would leave a repository as the only party able to say + /// where pnpm downloads a runtime from. pub(super) fn clear_workspace_project_fields(&mut self) { self.versioning = None; self.cargo = None; diff --git a/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs b/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs index 4fb02ba2fa..68885171f6 100644 --- a/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs +++ b/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs @@ -31,6 +31,7 @@ fn create_config( cache_dir: &Path, ) -> Config { Config { + tools: std::collections::BTreeMap::new(), bail: true, ci: false, update_notifier: true, diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs b/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs index 244631e579..4417d261be 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs @@ -39,11 +39,21 @@ pub enum BunResolverError { pub struct BunResolver { pub http_client: Arc, pub npm_resolver: Arc, + /// `tools.bun.mirror`: where Bun's releases are downloaded from. + pub mirror: Option, } impl BunResolver { pub fn new(http_client: Arc, npm_resolver: Arc) -> Self { - Self { http_client, npm_resolver } + Self { http_client, npm_resolver, mirror: None } + } + + /// Download Bun's releases from `tools.bun.mirror` rather than from + /// Bun's own. + #[must_use] + pub fn with_mirror(mut self, mirror: Option<&str>) -> Self { + self.mirror = mirror.map(ToString::to_string); + self } } @@ -91,7 +101,8 @@ impl BunResolver { as ResolveError })?; - let variants = read_bun_assets(&self.http_client, &version).await + let variants = read_bun_assets(&self.http_client, self.mirror.as_deref(), &version) + .await .map_err(|err| Box::new(BunResolverError::ReadAssets(err)) as ResolveError)?; let resolution = LockfileResolution::Variations(VariationsResolution { variants }); let manifest = serde_json::json!({ diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs index 85bde6da04..b25619a8ed 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs @@ -37,25 +37,36 @@ pub enum ReadBunAssetsError { /// Fetch and decode the Bun-release SHASUMS file for `version`. pub async fn read_bun_assets( http_client: &ThrottledClient, + mirror: Option<&str>, version: &str, ) -> Result, ReadBunAssetsError> { - let integrities_url = - format!("https://github.com/oven-sh/bun/releases/download/bun-v{version}/SHASUMS256.txt"); + let release = release_base(mirror, version); + let integrities_url = format!("{release}/SHASUMS256.txt"); let items = fetch_shasums_file(http_client, &integrities_url).await .map_err(ReadBunAssetsError::FetchShasumsFile)?; let mut variants = Vec::new(); for item in items { let Some(parsed) = parse_asset_name(&item.file_name) else { continue }; - variants.push(asset_resolution(version, &item, parsed)?); + variants.push(asset_resolution(&release, &item, parsed)?); } variants.sort_by(|a, b| variant_url(a).cmp(variant_url(b))); Ok(variants) } +/// Where one release's files are, under the mirror `tools.bun.mirror` +/// names or under Bun's own releases otherwise. Both lay a release out +/// the same way, so only the host above it differs. +fn release_base(mirror: Option<&str>, version: &str) -> String { + let base = mirror.map_or("https://github.com/oven-sh/bun/releases/download", |mirror| { + mirror.trim_end_matches('/') + }); + format!("{base}/bun-v{version}") +} + /// The download one `SHASUMS256.txt` entry describes. fn asset_resolution( - version: &str, + release: &str, item: &ShasumsFileItem, parsed: BunAssetName, ) -> Result { @@ -67,10 +78,7 @@ fn asset_resolution( error: Arc::new(error), })?; let binary = BinaryResolution { - url: format!( - "https://github.com/oven-sh/bun/releases/download/bun-v{version}/{file_name}", - file_name = item.file_name, - ), + url: format!("{release}/{file_name}", file_name = item.file_name), integrity, bin: BinarySpec::Single(bun_bin_path(&parsed.platform).to_string()), archive: BinaryArchive::Zip, diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs index e4bf884cf4..19917d8b37 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs @@ -1,6 +1,6 @@ use pretty_assertions::assert_eq; -use super::parse_asset_name; +use super::{parse_asset_name, release_base}; #[test] fn parses_apple_silicon_zip() { @@ -32,3 +32,21 @@ fn ignores_unrelated_assets() { assert!(parse_asset_name("bun-linux.zip").is_none()); assert!(parse_asset_name("bun-linux-x64.tar.gz").is_none()); } + +/// A release's checksums and its assets share one base, so both move +/// with the host. +#[test] +fn a_release_is_laid_out_the_same_under_a_mirror() { + assert_eq!( + release_base(None, "1.2.3"), + "https://github.com/oven-sh/bun/releases/download/bun-v1.2.3", + ); + assert_eq!( + release_base(Some("https://mirror.example.test/bun"), "1.2.3"), + "https://mirror.example.test/bun/bun-v1.2.3", + ); + assert_eq!( + release_base(Some("https://mirror.example.test/bun/"), "1.2.3"), + "https://mirror.example.test/bun/bun-v1.2.3", + ); +} diff --git a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs index 58585316e1..5e21334f85 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs @@ -11,18 +11,31 @@ pub const UNOFFICIAL_NODE_MIRROR_BASE_URL: &str = /// Resolve the base URL for a given release channel. /// -/// `node_download_mirrors` is the user's `.npmrc`/config override map -/// keyed by channel (`release`, `nightly`, `rc`, `test`, `v8-canary`). -/// A missing entry falls back to the official nodejs.org tree. The -/// returned URL always ends with `/` so callers can concatenate -/// `v/...` without a defensive check. +/// The three settings that can name it are read most specific first, +/// and the canonical spelling ahead of the older one where both are as +/// specific: +/// +/// 1. `channel`, from `tools.node.channels.`, which names this +/// channel and no other. +/// 2. `node_download_mirrors`, the older `node-mirror:` +/// spelling, kept because it has shipped. +/// 3. `mirror`, from `tools.node.mirror`, the base every channel hangs +/// off the way nodejs.org lays its own tree out. +/// +/// A channel none of them names falls back to the official nodejs.org +/// tree. The returned URL always ends with `/` so callers can +/// concatenate `v/...` without a defensive check. #[must_use] pub fn get_node_mirror( + mirror: Option<&str>, + channel: Option<&str>, node_download_mirrors: Option<&HashMap>, release_channel: &str, ) -> String { - let mirror = node_download_mirrors - .and_then(|map| map.get(release_channel).cloned()) + let mirror = channel + .map(ToString::to_string) + .or_else(|| node_download_mirrors.and_then(|map| map.get(release_channel).cloned())) + .or_else(|| mirror.map(|base| format!("{}/{release_channel}", base.trim_end_matches('/')))) .unwrap_or_else(|| format!("https://nodejs.org/download/{release_channel}/")); normalize_node_mirror(&mirror) } diff --git a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs index b437e32289..aeb1db76cf 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs @@ -14,20 +14,85 @@ fn configured_mirror_per_channel_wins_over_default() { ("v8-canary", "http://test.mirror.localhost/v8-canary"), ] { let mirrors = HashMap::from([(channel.to_string(), host.to_string())]); - assert_eq!(get_node_mirror(Some(&mirrors), channel), format!("{host}/")); + assert_eq!(get_node_mirror(None, None, Some(&mirrors), channel), format!("{host}/")); } } #[test] fn uses_defaults_when_unconfigured() { let empty = HashMap::new(); - assert_eq!(get_node_mirror(Some(&empty), "release"), "https://nodejs.org/download/release/"); - assert_eq!(get_node_mirror(None, "release"), "https://nodejs.org/download/release/"); + assert_eq!( + get_node_mirror(None, None, Some(&empty), "release"), + "https://nodejs.org/download/release/", + ); + assert_eq!( + get_node_mirror(None, None, None, "release"), + "https://nodejs.org/download/release/", + ); } #[test] fn appends_trailing_slash_when_missing() { let mirrors = HashMap::from([("release".to_string(), "http://test.mirror.localhost".to_string())]); - assert_eq!(get_node_mirror(Some(&mirrors), "release"), "http://test.mirror.localhost/"); + assert_eq!( + get_node_mirror(None, None, Some(&mirrors), "release"), + "http://test.mirror.localhost/", + ); +} + +/// nodejs.org lays its channels out below one base, which is the shape +/// a mirror of it carries. +#[test] +fn the_tool_mirror_is_the_base_every_channel_hangs_off() { + for channel in ["release", "nightly", "v8-canary"] { + assert_eq!( + get_node_mirror(Some("http://test.mirror.localhost/download"), None, None, channel), + format!("http://test.mirror.localhost/download/{channel}/"), + ); + } + assert_eq!( + get_node_mirror(Some("http://test.mirror.localhost/download/"), None, None, "release"), + "http://test.mirror.localhost/download/release/", + ); +} + +#[test] +fn a_channel_named_outright_wins_over_the_base() { + let mirrors = HashMap::from([("nightly".to_string(), "http://nightly.localhost".to_string())]); + assert_eq!( + get_node_mirror(Some("http://base.localhost"), None, Some(&mirrors), "nightly"), + "http://nightly.localhost/", + ); + assert_eq!( + get_node_mirror(Some("http://base.localhost"), None, Some(&mirrors), "release"), + "http://base.localhost/release/", + ); +} + +#[test] +fn a_channel_overrides_the_base_it_is_named_beside() { + assert_eq!( + get_node_mirror( + Some("http://base.localhost/download"), + Some("http://nightly.localhost"), + None, + "nightly", + ), + "http://nightly.localhost/", + ); + assert_eq!( + get_node_mirror(Some("http://base.localhost/download"), None, None, "release"), + "http://base.localhost/download/release/", + ); +} + +/// `node-mirror:` is the older spelling of the same thing. +#[test] +fn the_canonical_channel_wins_over_the_older_spelling() { + let mirrors = HashMap::from([("nightly".to_string(), "http://older.localhost".to_string())]); + assert_eq!( + get_node_mirror(None, Some("http://canonical.localhost"), Some(&mirrors), "nightly"), + "http://canonical.localhost/", + ); } diff --git a/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs b/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs index 213687e435..594fcb68c1 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs @@ -97,6 +97,11 @@ pub struct NodeResolver { pub http_client: Arc, pub auth_headers: Arc, pub node_download_mirrors: HashMap, + /// `tools.node.mirror`: the base every release channel hangs off. + pub mirror: Option, + /// `tools.node.channels`: where one release channel comes from when + /// it does not come from the same place as the rest. + pub channel_mirrors: HashMap, pub offline: bool, /// The pnpm cache directory backing the per-version SHASUMS disk /// cache. `None` disables the cache and every resolve fetches the @@ -119,6 +124,8 @@ impl NodeResolver { http_client, auth_headers, node_download_mirrors: HashMap::new(), + mirror: None, + channel_mirrors: HashMap::new(), offline: false, cache_dir: None, } @@ -240,7 +247,12 @@ impl NodeResolver { } let parsed = parse_node_specifier(version_spec).map_err(NodeResolverError::InvalidReleaseChannel)?; - let mirror = get_node_mirror(Some(&self.node_download_mirrors), &parsed.release_channel); + let mirror = get_node_mirror( + self.mirror.as_deref(), + self.channel_mirrors.get(&parsed.release_channel).map(String::as_str), + Some(&self.node_download_mirrors), + &parsed.release_channel, + ); if let Some(version) = exact_release_version(&parsed) { return Ok(PickedNodeVersion { version, @@ -312,7 +324,12 @@ impl NodeResolver { .map_err(|err| { Box::new(NodeResolverError::InvalidReleaseChannel(err)) as ResolveError })?; - let mirror = get_node_mirror(Some(&self.node_download_mirrors), &parsed.release_channel); + let mirror = get_node_mirror( + self.mirror.as_deref(), + self.channel_mirrors.get(&parsed.release_channel).map(String::as_str), + Some(&self.node_download_mirrors), + &parsed.release_channel, + ); let version = resolve_node_version_with_auth( &self.http_client, &self.auth_headers, diff --git a/pnpm/crates/package-manager/src/add/specifier.rs b/pnpm/crates/package-manager/src/add/specifier.rs index 3f4cc3e779..cb817188d0 100644 --- a/pnpm/crates/package-manager/src/add/specifier.rs +++ b/pnpm/crates/package-manager/src/add/specifier.rs @@ -6,7 +6,7 @@ use super::{ }; use crate::{CatalogModeDep, decide_catalog_outcome}; use pnpm_catalogs_types::Catalogs; -use pnpm_config::{Config, SaveWorkspaceProtocol}; +use pnpm_config::{Config, SaveWorkspaceProtocol, Tool}; use pnpm_engine_runtime_node_resolver::NodeResolver; use pnpm_package_manifest::{DependencyGroup, PackageManifest}; use pnpm_package_name::is_valid_dependency_alias; @@ -199,6 +199,8 @@ pub(super) async fn resolve_node_runtime_specifier( std::sync::Arc::clone(&config.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&config.node_download_mirrors); + node_resolver.mirror = config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = config.tool_channel_mirrors(Tool::Node); node_resolver.offline = config.offline; node_resolver.cache_dir = Some(config.cache_dir.clone()); node_resolver diff --git a/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs b/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs index 4aec877a6f..6e6f52bf15 100644 --- a/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs +++ b/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs @@ -8,7 +8,7 @@ use super::InstallWithFreshLockfileError; use crate::{PrefetchContext, PrefetchingResolver}; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -339,6 +339,8 @@ impl ResolverChainInputs<'_> { Arc::clone(self.fetching.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&self.config.node_download_mirrors); + node_resolver.mirror = self.config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = self.config.tool_channel_mirrors(Tool::Node); node_resolver.offline = self.config.offline; node_resolver.cache_dir = Some(self.config.cache_dir.clone()); node_resolver @@ -407,10 +409,10 @@ impl ResolverChainInputs<'_> { Arc::clone(self.fetching.http_client), Arc::clone(npm_resolver), )), - Box::new(BunResolver::new( - Arc::clone(self.fetching.http_client), - Arc::clone(npm_resolver), - )), + Box::new( + BunResolver::new(Arc::clone(self.fetching.http_client), Arc::clone(npm_resolver)) + .with_mirror(self.config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new( Arc::clone(self.fetching.http_client), self.config.tls.strict_ssl.unwrap_or(true), diff --git a/pnpm/crates/package-manager/src/update/latest.rs b/pnpm/crates/package-manager/src/update/latest.rs index 4ab7b5a4c8..f692aeb395 100644 --- a/pnpm/crates/package-manager/src/update/latest.rs +++ b/pnpm/crates/package-manager/src/update/latest.rs @@ -8,7 +8,7 @@ use crate::{ }; use chrono::{DateTime, Utc}; use node_semver::Version; -use pnpm_config::{Config, version_policy::PackageVersionPolicy}; +use pnpm_config::{Config, Tool, version_policy::PackageVersionPolicy}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -180,13 +180,18 @@ pub(super) fn ensure_latest_resolver_chain<'chain>( Arc::clone(&ctx.config.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&ctx.config.node_download_mirrors); + node_resolver.mirror = ctx.config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = ctx.config.tool_channel_mirrors(Tool::Node); node_resolver.offline = ctx.config.offline; node_resolver.cache_dir = Some(ctx.config.cache_dir.clone()); let resolver = DefaultResolver::new(vec![ Box::new(Arc::clone(&npm_resolver)) as Box, Box::new(node_resolver), Box::new(DenoResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver))), - Box::new(BunResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver))), + Box::new( + BunResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver)) + .with_mirror(ctx.config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new( Arc::clone(ctx.http_client_arc), ctx.config.tls.strict_ssl.unwrap_or(true), diff --git a/pnpm/crates/python-installer/src/interpreter/download.rs b/pnpm/crates/python-installer/src/interpreter/download.rs index a38cb3d71d..9cd181327e 100644 --- a/pnpm/crates/python-installer/src/interpreter/download.rs +++ b/pnpm/crates/python-installer/src/interpreter/download.rs @@ -9,7 +9,7 @@ use super::{InterpreterCommand, VersionRequest, command::interpreter_in}; use miette::{IntoDiagnostic, Result, WrapErr, bail}; -use pnpm_config::{Config, RuntimeOnFail}; +use pnpm_config::{Config, DEFAULT_PYTHON_DOWNLOAD_URL, RuntimeOnFail, Tool}; use pnpm_crypto_shasums_file::{ShasumsFileItem, fetch_moving_shasums_file_cached}; use pnpm_network::{AuthHeaders, ThrottledClient}; use pnpm_reporter::{GlobalLog, LogEvent, LogLevel, Reporter}; @@ -56,10 +56,18 @@ pub(super) struct Build { integrity: ssri::Integrity, } +/// Where the interpreter builds are downloaded from: the mirror +/// `tools.python.mirror` names, or python-build-standalone's own +/// releases. A mirror lays a release out the way that project does, so +/// only the host above it differs. +fn releases_url(config: &Config) -> &str { + config.tool_mirror(Tool::Python).unwrap_or(DEFAULT_PYTHON_DOWNLOAD_URL) +} + impl Releases { /// The interpreters pnpm can install. pub(super) async fn read(config: &Config, client: &ThrottledClient) -> Result { - let url = format!("{}/latest/download/SHA256SUMS", config.python.download_url); + let url = format!("{}/latest/download/SHA256SUMS", releases_url(config)); let index = fetch_moving_shasums_file_cached( client, &url, @@ -129,7 +137,7 @@ impl Build { config: &Config, client: &ThrottledClient, ) -> Result { - let url = format!("{}/download/{}/{}", config.python.download_url, self.tag, self.file); + let url = format!("{}/download/{}/{}", releases_url(config), self.tag, self.file); let response = client .get_limited_bytes_with_secure_auth_and_retry( &url, diff --git a/pnpm/crates/resolving-default-resolver/src/standalone.rs b/pnpm/crates/resolving-default-resolver/src/standalone.rs index 748d660422..4419c2bb35 100644 --- a/pnpm/crates/resolving-default-resolver/src/standalone.rs +++ b/pnpm/crates/resolving-default-resolver/src/standalone.rs @@ -20,7 +20,7 @@ //! install-time concern. use crate::DefaultResolver; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -84,7 +84,10 @@ pub fn build_standalone_chain( Box::new(LocalSchemeResolver::new(local_ctx)), Box::new(build_node_resolver(config, http_client)), Box::new(DenoResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver))), - Box::new(BunResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver))), + Box::new( + BunResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver)) + .with_mirror(config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new(Arc::clone(http_client), config.tls.strict_ssl.unwrap_or(true))), Box::new(build_named_registry_resolver(opts, retry_opts)?), Box::new(LocalPathResolver::new(local_ctx)), @@ -148,6 +151,8 @@ fn build_node_resolver(config: &Config, http_client: &Arc) -> N let mut node_resolver = NodeResolver::new_with_auth(Arc::clone(http_client), Arc::clone(&config.auth_headers)); node_resolver.node_download_mirrors.clone_from(&config.node_download_mirrors); + node_resolver.mirror = config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = config.tool_channel_mirrors(Tool::Node); node_resolver.offline = config.offline; node_resolver.cache_dir = Some(config.cache_dir.clone()); node_resolver