diff --git a/.changeset/install-a-python-interpreter.md b/.changeset/install-a-python-interpreter.md index 891148fdea..1799f8799d 100644 --- a/.changeset/install-a-python-interpreter.md +++ b/.changeset/install-a-python-interpreter.md @@ -2,4 +2,4 @@ "pacquet": minor --- -`pnpm install` now installs a Python interpreter when no interpreter on the machine fits the project [#14945](https://github.com/pnpm/pnpm/issues/14945). The builds are [python-build-standalone](https://github.com/astral-sh/python-build-standalone)'s, which uv and rye install too. One interpreter is shared by every project on the machine, and a later install uses it without downloading anything. `runtimeOnFail` decides what an install with no interpreter that fits does, the way it does for a Node.js runtime. `error` reports the project instead of installing one. `warn` and `ignore` install with an interpreter the machine has that the project's `requires-python` rejects. `python.downloadUrl` names a mirror. +`pnpm install` now installs a Python interpreter when no interpreter on the machine fits the project [#14945](https://github.com/pnpm/pnpm/issues/14945). The builds are [python-build-standalone](https://github.com/astral-sh/python-build-standalone)'s, which uv and rye install too. One interpreter is shared by every project on the machine, and a later install uses it without downloading anything. `runtimeOnFail` decides what an install with no interpreter that fits does, the way it does for a Node.js runtime. `error` reports the project instead of installing one. `warn` and `ignore` install with an interpreter the machine has that the project's `requires-python` rejects. `tools.python.mirror` names a mirror. diff --git a/.changeset/tools-mirror.md b/.changeset/tools-mirror.md new file mode 100644 index 0000000000..c3a0b61bc2 --- /dev/null +++ b/.changeset/tools-mirror.md @@ -0,0 +1,27 @@ +--- +"pacquet": minor +--- + +`tools` names the programs pnpm downloads, and `mirror` says where each one comes from. + +```yaml +tools: + node: + mirror: https://mirror.example.com/node/download + channels: + nightly: https://nightly.example.com/ + bun: + mirror: https://mirror.example.com/bun + python: + mirror: https://mirror.example.com/python-build-standalone/releases +``` + +`node`, `bun` and `python` can be named. Any other tool is refused. + +`mirror` is the base a tool's own layout hangs off. + +`channels` sends one release channel elsewhere. A channel neither it nor `node-mirror:` names is left to `mirror`. Only `node` publishes channels, so naming them for another tool is refused. + +Set it in the global `config.yaml` or in `PNPM_CONFIG_TOOLS`. A `pnpm-workspace.yaml` that names a tool mirror is ignored. + +`pnpm pack-app` downloads the Node.js it embeds through `tools.node`. `node-mirror:` keeps working and names the same thing as an entry under `channels`. diff --git a/pnpm/crates/cli/src/cli_args/env.rs b/pnpm/crates/cli/src/cli_args/env.rs index 0575cdb0e2..677e53cfb9 100644 --- a/pnpm/crates/cli/src/cli_args/env.rs +++ b/pnpm/crates/cli/src/cli_args/env.rs @@ -5,7 +5,7 @@ use super::{global::handle_global_add, registry_client::build_registry_client}; use clap::Args; use derive_more::{Display, Error}; use miette::Diagnostic; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_runtime_node_resolver::{ get_node_mirror, parse_node_specifier, resolve_node_versions_with_auth, }; @@ -145,8 +145,13 @@ impl EnvArgs { pub async fn run_list(version_spec: Option, config: &Config) -> miette::Result { let specifier = parse_node_specifier(version_spec.as_deref().unwrap_or_default()) .map_err(miette::Report::new)?; - let mirror = - get_node_mirror(Some(&config.node_download_mirrors), &specifier.release_channel); + let channels = config.tool_channel_mirrors(Tool::Node); + let mirror = get_node_mirror( + config.tool_mirror(Tool::Node), + channels.get(&specifier.release_channel).map(String::as_str), + Some(&config.node_download_mirrors), + &specifier.release_channel, + ); let http_client = build_registry_client(config)?; let mut versions = resolve_node_versions_with_auth( &http_client, diff --git a/pnpm/crates/cli/src/cli_args/pack_app.rs b/pnpm/crates/cli/src/cli_args/pack_app.rs index fd58d52de9..b215cdab4a 100644 --- a/pnpm/crates/cli/src/cli_args/pack_app.rs +++ b/pnpm/crates/cli/src/cli_args/pack_app.rs @@ -19,9 +19,9 @@ //! home. use build::{ - SeaBuild, ad_hoc_sign_mac_binary, ensure_node_runtime, pnpm_home_dir, print_built, - reject_non_regular_output_file, reject_non_regular_outputs, resolve_builder_binary, - resolve_version, run_command, + EmbeddedRuntime, SeaBuild, ad_hoc_sign_mac_binary, ensure_node_runtime, pnpm_home_dir, + print_built, reject_non_regular_output_file, reject_non_regular_outputs, + resolve_builder_binary, resolve_version, run_command, }; use clap::Args; use config::{ @@ -302,14 +302,16 @@ impl PackAppArgs { // the serialized format has changed across Node.js minor releases, // so a blob produced by a builder of a different version than the // embedded runtime fails deserialization at startup. - let target_version = resolve_version(config, &requested_node_spec).await?; + let runtime = EmbeddedRuntime { + build_root, + version: resolve_version(config, &requested_node_spec).await?, + }; let build = SeaBuild { - builder_bin: resolve_builder_binary(&build_root, &target_version)?, + builder_bin: resolve_builder_binary(&runtime)?, pacquet_bin: std::env::current_exe() .into_diagnostic() .wrap_err("resolving the pnpm executable path")?, - build_root, - target_version, + runtime, dir, output_dir, output_name, @@ -446,8 +448,7 @@ impl SeaBuild<'_> { fn build_target(&self, target: &ParsedTarget) -> miette::Result { let embedded_node_bin = ensure_node_runtime( &self.pacquet_bin, - &self.build_root, - &self.target_version, + &self.runtime, &target.platform, &target.arch, target.libc.as_deref(), @@ -478,14 +479,12 @@ impl SeaBuild<'_> { self.build_sea(&output_file, &embedded_node_bin)?; ad_hoc_sign_mac_binary(target, &output_file, self.dir)?; - Ok( - format!( - " {}: {} (Node.js {})", - target.raw, - output_file.display(), - self.target_version, - ), - ) + Ok(format!( + " {}: {} (Node.js {})", + target.raw, + output_file.display(), + self.runtime.version, + )) } fn build_sea(&self, output_file: &Path, embedded_node_bin: &Path) -> miette::Result<()> { let sea_config = serde_json::json!({ diff --git a/pnpm/crates/cli/src/cli_args/pack_app/build.rs b/pnpm/crates/cli/src/cli_args/pack_app/build.rs index cc0fa9728c..fd825ad7f5 100644 --- a/pnpm/crates/cli/src/cli_args/pack_app/build.rs +++ b/pnpm/crates/cli/src/cli_args/pack_app/build.rs @@ -10,12 +10,18 @@ pub(super) struct SeaBuild<'a> { pub(super) output_dir: PathBuf, pub(super) output_name: String, pub(super) entry: PathBuf, - pub(super) build_root: PathBuf, - pub(super) target_version: String, + pub(super) runtime: EmbeddedRuntime, pub(super) builder_bin: PathBuf, pub(super) pacquet_bin: PathBuf, } +/// The Node.js the executables embed: which build, and where the copies +/// of it are kept. +pub(super) struct EmbeddedRuntime { + pub(super) build_root: PathBuf, + pub(super) version: String, +} + /// Reject a pre-existing symlink (or any non-regular file) at any /// target's final output path before downloading anything: a repo /// could commit `dist-app//` as a symlink pointing @@ -49,13 +55,11 @@ pub(super) fn print_built(results: &[String]) { /// Unlike pnpm — which reuses its own running interpreter when it already /// matches — pacquet has no host Node.js, so it always downloads a /// host-arch Node.js of the target version. -pub(super) fn resolve_builder_binary( - build_root: &Path, - target_version: &str, -) -> miette::Result { +pub(super) fn resolve_builder_binary(runtime: &EmbeddedRuntime) -> miette::Result { + let target_version = &runtime.version; if !builder_version_can_build_sea(target_version) { return Err(PackAppError::RuntimeTooOld { - version: target_version.to_string(), + version: target_version.clone(), major: MIN_BUILDER_VERSION.0, minor: MIN_BUILDER_VERSION.1, } @@ -65,8 +69,7 @@ pub(super) fn resolve_builder_binary( std::env::current_exe().into_diagnostic().wrap_err("resolving the pnpm executable path")?; ensure_node_runtime( &pacquet_bin, - build_root, - target_version, + runtime, pnpm_detect_libc::host_platform(), pnpm_detect_libc::host_arch(), // Pin libc to the host's. Otherwise a caller that set @@ -106,12 +109,12 @@ fn builder_version_can_build_sea(version: &str) -> bool { /// directory. pub(super) fn ensure_node_runtime( pacquet_bin: &Path, - build_root: &Path, - version: &str, + runtime: &EmbeddedRuntime, platform: &str, arch: &str, libc: Option<&str>, ) -> miette::Result { + let EmbeddedRuntime { build_root, version } = runtime; // Linux variants always need a libc pin (glibc or musl) so variant // selection is deterministic and doesn't depend on the host's detected // libc or the user's supportedArchitectures.libc config. @@ -146,7 +149,7 @@ pub(super) fn ensure_node_runtime( if !binary_path.exists() { return Err(PackAppError::NodeBinaryMissing { path: binary_path.display().to_string(), - version: version.to_string(), + version: version.clone(), } .into()); } @@ -159,10 +162,18 @@ fn node_binary_path(node_dir: &Path, platform: &str) -> PathBuf { pub(super) async fn resolve_version(config: &Config, specifier: &str) -> miette::Result { let parsed = parse_node_specifier(specifier).map_err(miette::Report::new)?; - // pacquet has no `node-download-mirrors` config field yet, so the - // override map is always absent and the official nodejs.org tree is - // used. Matches pnpm's default when `nodeDownloadMirrors` is unset. - let mirror = get_node_mirror(None, &parsed.release_channel); + // `node-mirror:` is deliberately not read here. A workspace + // may name it, and this runtime is executed as the builder and kept + // in the shared pack-app cache, so a repository naming it would be + // choosing the program that packs everyone's app. `tools` carries no + // such risk, being the machine's own. + let channels = config.tool_channel_mirrors(pnpm_config::Tool::Node); + let mirror = get_node_mirror( + config.tool_mirror(pnpm_config::Tool::Node), + channels.get(&parsed.release_channel).map(String::as_str), + None, + &parsed.release_channel, + ); let http_client = build_http_client(config)?; let version = resolve_node_version(&http_client, &parsed.version_specifier, Some(&mirror)) .await diff --git a/pnpm/crates/cli/tests/suite/python.rs b/pnpm/crates/cli/tests/suite/python.rs index 7f75cea22d..4693ecc387 100644 --- a/pnpm/crates/cli/tests/suite/python.rs +++ b/pnpm/crates/cli/tests/suite/python.rs @@ -189,6 +189,78 @@ fn add_python_settings(root: &Path, settings: &str) { .unwrap(); } +/// A repository that named a mirror would be choosing which program runs +/// on the machine of everyone who clones it, and a release's checksums +/// come from the mirror that serves its files, so the download verifies +/// only that the mirror agrees with itself. +/// +/// The machine and the repository name mirrors serving different +/// releases, and only the repository's serves the pinned one. Honouring +/// the workspace would install it, so the install failing is what says +/// the workspace was not read. The machine's mirror is named in the +/// global `config.yaml` rather than the environment, because the +/// environment outranks the workspace either way and would prove +/// nothing. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn a_repository_cannot_name_a_mirror() { + use command_extra::CommandExtra as _; + + let root = tempfile::tempdir().unwrap(); + let mut machine = mockito::Server::new_async().await; + let mut repository = mockito::Server::new_async().await; + project(root.path(), "https://unused.invalid", &[]); + let machines = serve_interpreter(&mut machine, &["3.13.90"]).await; + let pinned = serve_interpreter(&mut repository, &["3.13.95"]).await; + + let config = root.path().join(".config/pnpm"); + fs::create_dir_all(&config).unwrap(); + fs::write( + config.join("config.yaml"), + format!("tools:\n python:\n mirror: '{}'\n", machine.url()), + ) + .unwrap(); + let path = root.path().join("pnpm-workspace.yaml"); + let mut workspace = fs::read_to_string(&path).unwrap(); + writeln!(workspace, "tools:\n python:\n mirror: '{}'", repository.url()).unwrap(); + fs::write(&path, workspace).unwrap(); + fs::write( + root.path().join("pyproject.toml"), + "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.95'\ndependencies = []\n", + ) + .unwrap(); + + assert_failure_contains( + pacquet_in(root.path()) + .with_env("XDG_CONFIG_HOME", root.path().join(".config")) + .arg("install"), + "3.13.95", + ); + for mock in pinned { + assert!(!mock.matched_async().await, "the repository's mirror was read"); + } + // Without this the test would also pass if the machine's mirror were + // never found either: the install would reach for the real releases, + // fail naming the same version, and leave the repository's untouched. + let mut read = false; + for mock in machines { + read |= mock.matched_async().await; + } + assert!(read, "the machine's mirror was not read"); +} + +/// A pnpm that downloads interpreters from `mirror`. +/// +/// A tool mirror is the machine's to name, so a test names one the way a +/// user does, through the environment, rather than through the workspace +/// file the run would ignore it in. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn pacquet_with_python_mirror(root: &Path, mirror: &str) -> Command { + let mut command = pacquet_in(root); + command.env("PNPM_CONFIG_TOOLS", format!(r#"{{"python":{{"mirror":"{mirror}"}}}}"#)); + command +} + fn add_supported_architectures(root: &Path, platforms: &[&str]) { let path = root.join("pnpm-workspace.yaml"); let mut workspace = fs::read_to_string(&path).unwrap(); @@ -554,14 +626,14 @@ async fn installs_an_interpreter_no_machine_has_and_reuses_it() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let release = serve_interpreter(&mut server, &["3.13.99"]).await; - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); + let mirror = server.url(); let install = || { fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.99'\ndependencies = []\n", ) .unwrap(); - pacquet_in(root.path()) + pacquet_with_python_mirror(root.path(), &mirror) }; install() @@ -598,15 +670,15 @@ async fn a_pin_the_release_moved_past_installs_the_version_it_has() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.96"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '>=3.13.90,<3.14'\ndependencies = []\n", ) .unwrap(); fs::write(root.path().join(".python-version"), "3.13.95\n").unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); - let output = pacquet_in(root.path()) + let output = pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .output() .unwrap(); @@ -626,15 +698,15 @@ async fn a_pin_the_project_refuses_installs_a_version_it_accepts() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.94", "3.13.93"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '>=3.13.94,<3.14'\ndependencies = []\n", ) .unwrap(); fs::write(root.path().join(".python-version"), "3.13.93\n").unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); - let output = pacquet_in(root.path()) + let output = pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .output() .unwrap(); @@ -667,14 +739,14 @@ async fn refuses_an_interpreter_the_release_does_not_name() { .with_body("not the interpreter the release names") .create_async() .await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.97'\ndependencies = []\n", ) .unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); assert_failure_contains( - pacquet_in(root.path()).arg("install"), + pacquet_with_python_mirror(root.path(), &mirror).arg("install"), "is not the one the release names", ); assert!( @@ -695,21 +767,22 @@ async fn an_interpreter_is_installed_only_where_the_install_may_download() { let mut server = mockito::Server::new_async().await; project(root.path(), "https://unused.invalid", &[]); let _release = serve_interpreter(&mut server, &["3.13.98"]).await; + let mirror = server.url(); fs::write( root.path().join("pyproject.toml"), "[project]\nname = 'app'\nversion = '1.0'\nrequires-python = '==3.13.98'\ndependencies = []\n", ) .unwrap(); - add_python_settings(root.path(), &format!(" downloadUrl: '{}'\n", server.url())); assert_failure_contains( - pacquet_in(root.path()).args(["install", "--runtime-on-fail=error"]), + pacquet_with_python_mirror(root.path(), &mirror) + .args(["install", "--runtime-on-fail=error"]), "no Python interpreter for", ); assert_failure_contains( - pacquet_in(root.path()).args(["install", "--offline"]), + pacquet_with_python_mirror(root.path(), &mirror).args(["install", "--offline"]), "no Python interpreter for", ); - pacquet_in(root.path()) + pacquet_with_python_mirror(root.path(), &mirror) .arg("install") .assert() .success(); diff --git a/pnpm/crates/config/src/env_overlay.rs b/pnpm/crates/config/src/env_overlay.rs index 9d14b7ae15..db87aa3db6 100644 --- a/pnpm/crates/config/src/env_overlay.rs +++ b/pnpm/crates/config/src/env_overlay.rs @@ -363,6 +363,7 @@ impl WorkspaceSettings { ); json_field!(settings, Sys, changed_files_ignore_pattern, "CHANGED_FILES_IGNORE_PATTERN"); json_field!(settings, Sys, supported_architectures, "SUPPORTED_ARCHITECTURES"); + json_field!(settings, Sys, tools, "TOOLS"); json_field!(settings, Sys, ignored_optional_dependencies, "IGNORED_OPTIONAL_DEPENDENCIES"); json_field!(settings, Sys, overrides, "OVERRIDES"); json_field!(settings, Sys, package_extensions, "PACKAGE_EXTENSIONS"); diff --git a/pnpm/crates/config/src/known_settings.rs b/pnpm/crates/config/src/known_settings.rs index 9b8b2815a2..be198510b1 100644 --- a/pnpm/crates/config/src/known_settings.rs +++ b/pnpm/crates/config/src/known_settings.rs @@ -47,6 +47,7 @@ const TYPED_WORKSPACE_MANIFEST_KEYS: &[&str] = &[ "requiredScripts", "sideEffectsCache", "supportedArchitectures", + "tools", "update", "updateConfig", "versioning", diff --git a/pnpm/crates/config/src/lib.rs b/pnpm/crates/config/src/lib.rs index 044c283c8d..ff083e7ecc 100644 --- a/pnpm/crates/config/src/lib.rs +++ b/pnpm/crates/config/src/lib.rs @@ -38,8 +38,8 @@ pub use workspace_yaml::{ AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, GLOBAL_CONFIG_YAML_FILENAME, LoadWorkspaceYamlError, PackageExtension, PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, PythonSettings, RemoteSideEffectsCacheSettings, - TaskSettings, UpdateConfig, UpdateSettings, WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, - WorkspaceSettings, decided_allow_builds, + TaskSettings, Tool, ToolSettings, UpdateConfig, UpdateSettings, WORKSPACE_MANIFEST_FILENAME, + WorkspaceKeyIssues, WorkspaceSettings, decided_allow_builds, package_configs::{self, PackageConfigsSetting, ProjectConfig, ProjectConfigMultiMatch}, registries::{self, RegistryDeclaration, RegistryEntry, RegistryLookups}, workspace_root_or, diff --git a/pnpm/crates/config/src/settings.rs b/pnpm/crates/config/src/settings.rs index eaaf68b75d..d75d9a15f1 100644 --- a/pnpm/crates/config/src/settings.rs +++ b/pnpm/crates/config/src/settings.rs @@ -5,8 +5,8 @@ use super::{ PackageManagerBootstrap, PatchGroupRecord, PatchInput, Path, PathBuf, Pipe, PmOnFail, ProjectConfig, PythonSettings, RegistryOptions, RemoteSideEffectsCacheSettings, ResolutionMode, ResolvePatchedDependenciesError, RuntimeOnFail, SaveWorkspaceProtocol, ScriptsPrependNodePath, - SmartDefault, StoreDir, TrustPolicy, VerifyDepsBeforeRun, WorkspaceKeyIssues, - create_hex_hash_from_file, default_cache_dir, default_child_concurrency, + SmartDefault, StoreDir, Tool, ToolSettings, TrustPolicy, VerifyDepsBeforeRun, + WorkspaceKeyIssues, create_hex_hash_from_file, default_cache_dir, default_child_concurrency, default_enable_global_virtual_store, default_fetch_min_speed_ki_bps, default_fetch_retries, default_fetch_retry_factor, default_fetch_retry_maxtimeout, default_fetch_retry_mintimeout, default_fetch_timeout, default_fetch_warn_timeout_ms, default_git_shallow_hosts, @@ -971,6 +971,8 @@ pub struct Config { /// Cargo dependency management declared by the workspace. pub cargo: CargoSettings, pub python: PythonSettings, + /// `tools` from `pnpm-workspace.yaml`, keyed by tool name. + pub tools: BTreeMap, pub remote_side_effects_cache: Option, @@ -1695,6 +1697,37 @@ pub struct Config { } impl Config { + /// Where the builds of one tool are downloaded from, as + /// `tools..mirror` names it, without the trailing slash a + /// caller joins onto. + #[must_use] + pub fn tool_mirror(&self, tool: Tool) -> Option<&str> { + self.tools + .get(&tool)? + .mirror + .as_deref() + .map(|mirror| mirror.trim_end_matches('/')) + } + + /// Where each line of a tool's builds is downloaded from, as + /// `tools..channels` names them. A line it does not name is + /// left to [`Self::tool_mirror`]. + #[must_use] + pub fn tool_channel_mirrors(&self, tool: Tool) -> HashMap { + self.tools + .get(&tool) + .and_then(|tool| tool.channels.as_ref()) + .map(|channels| { + channels + .iter() + .map(|(channel, mirror)| { + (channel.clone(), mirror.trim_end_matches('/').to_string()) + }) + .collect() + }) + .unwrap_or_default() + } + #[must_use] pub fn new() -> Self { Self::default() diff --git a/pnpm/crates/config/src/workspace_settings.rs b/pnpm/crates/config/src/workspace_settings.rs index e20fffa3dd..031087bc74 100644 --- a/pnpm/crates/config/src/workspace_settings.rs +++ b/pnpm/crates/config/src/workspace_settings.rs @@ -99,6 +99,14 @@ impl Config { settings.scope = None; settings.global_dir = None; settings.global_bin_dir = None; + // A mirror decides where the binary pnpm runs an ecosystem with is + // downloaded from, and a release's checksums come from the mirror + // that serves its files, so verifying the download says only that + // the mirror agrees with itself. A repository that named one would + // be choosing which program runs on the machine of everyone who + // clones it. `tools` therefore comes from the global `config.yaml` + // and `PNPM_CONFIG_TOOLS` only. + settings.tools = None; // Noted rather than assigned, so an `enableGlobalVirtualStore` / // `virtualStoreDir` set in the global `config.yaml` still counts as // "explicitly set" when the workspace yaml leaves it unset. diff --git a/pnpm/crates/config/src/workspace_yaml.rs b/pnpm/crates/config/src/workspace_yaml.rs index 8041c84014..79e209cc42 100644 --- a/pnpm/crates/config/src/workspace_yaml.rs +++ b/pnpm/crates/config/src/workspace_yaml.rs @@ -1,11 +1,12 @@ pub mod package_configs; pub mod registries; pub use error::LoadWorkspaceYamlError; +pub(crate) use sections::deserialize_tools; pub use sections::{ AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, PackageExtension, PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, PythonSettings, RemoteSideEffectsCacheSettings, SideEffectsCacheSetting, SideEffectsCacheSettings, - TaskSettings, UpdateConfig, UpdateSettings, decided_allow_builds, + TaskSettings, Tool, ToolSettings, UpdateConfig, UpdateSettings, decided_allow_builds, }; pub use settings::WorkspaceSettings; @@ -75,6 +76,28 @@ fn overlay(target: &mut Setting, value: Option) { } } +/// [`overlay`] for the tools, which every layer answers for separately. +/// +/// The tools are independent of one another, so a workspace naming a Bun +/// mirror has said nothing about Node.js and must not drop the one the +/// machine's own config names. The same holds a level down: a layer that +/// names a tool's release channels has not said where the rest of its +/// builds come from. +fn overlay_tools( + target: &mut BTreeMap, + value: Option>, +) { + for (tool, named) in value.into_iter().flatten() { + let settings = target.entry(tool).or_default(); + overlay_some(&mut settings.mirror, named.mirror); + match (&mut settings.channels, named.channels) { + (Some(channels), Some(named)) => channels.extend(named), + (channels @ None, named @ Some(_)) => *channels = named, + (_, None) => {} + } + } +} + /// [`overlay`] for a target that is itself optional: an unset field leaves /// whatever the previous layer recorded, including its absence. fn overlay_some(target: &mut Option, value: Option) { diff --git a/pnpm/crates/config/src/workspace_yaml/apply.rs b/pnpm/crates/config/src/workspace_yaml/apply.rs index 8da5a9072c..f36ea5c608 100644 --- a/pnpm/crates/config/src/workspace_yaml/apply.rs +++ b/pnpm/crates/config/src/workspace_yaml/apply.rs @@ -1,7 +1,8 @@ use super::{ Config, Path, PnpmfileSetting, ProxyKeys, ProxyValue, SideEffectsCacheSetting, StoreDir, UpdateConfig, WorkspaceSettings, decided_allow_builds, no_proxy_scalar, normalize_registry_url, - overlay, overlay_some, registries, resolve, resolve_child_concurrency, warn_deprecated_pairing, + overlay, overlay_some, overlay_tools, registries, resolve, resolve_child_concurrency, + warn_deprecated_pairing, }; impl WorkspaceSettings { @@ -164,6 +165,7 @@ impl WorkspaceSettings { overlay_some(&mut config.pnpr_server, self.pnpr_server.take()); overlay(&mut config.cargo, self.cargo.take()); overlay(&mut config.python, self.python.take()); + overlay_tools(&mut config.tools, self.tools.take()); if let Some(v) = self.remote_side_effects_cache.take() { config.remote_side_effects_cache.get_or_insert_default().overlay(v); } diff --git a/pnpm/crates/config/src/workspace_yaml/reset.rs b/pnpm/crates/config/src/workspace_yaml/reset.rs index 35d05e0eb8..7c851bcd4b 100644 --- a/pnpm/crates/config/src/workspace_yaml/reset.rs +++ b/pnpm/crates/config/src/workspace_yaml/reset.rs @@ -122,7 +122,7 @@ impl WorkspaceSettings { global_pnpmfile, pnpmfile, global_dir, global_bin_dir, cache_dir, prefer_frozen_lockfile, lockfile, merge_git_branch_lockfiles, optimistic_repeat_install, minimum_release_age, global_shims, frozen_lockfile, - registry, scope, pnpr_server, cargo, python, remote_side_effects_cache, + registry, scope, pnpr_server, cargo, python, tools, remote_side_effects_cache, reporter_hide_prefix, max_sockets, patched_dependencies, patches_dir, config_dependencies, dangerously_allow_all_builds, strict_dep_builds, ignore_scripts, ignore_pnpmfile, git_checks, engine_strict, node_version, diff --git a/pnpm/crates/config/src/workspace_yaml/resolved.rs b/pnpm/crates/config/src/workspace_yaml/resolved.rs index a9ab69f043..f7de610fce 100644 --- a/pnpm/crates/config/src/workspace_yaml/resolved.rs +++ b/pnpm/crates/config/src/workspace_yaml/resolved.rs @@ -107,6 +107,7 @@ impl WorkspaceSettings { pnpr_server: config.pnpr_server.clone(), cargo: Some(config.cargo.clone()), python: Some(config.python.clone()), + tools: Some(config.tools.clone()), remote_side_effects_cache: config.remote_side_effects_cache.clone(), reporter_hide_prefix: config.reporter_hide_prefix, max_sockets: config.max_sockets, diff --git a/pnpm/crates/config/src/workspace_yaml/sections.rs b/pnpm/crates/config/src/workspace_yaml/sections.rs index a6840ff683..10c39195a4 100644 --- a/pnpm/crates/config/src/workspace_yaml/sections.rs +++ b/pnpm/crates/config/src/workspace_yaml/sections.rs @@ -99,6 +99,64 @@ pub struct RemoteSideEffectsCacheSettings { pub private_key: Option, } +/// A program pnpm downloads through a base URL, which is what a mirror +/// can replace. +/// +/// Deno and Yarn are absent on purpose: both read the GitHub API for +/// their release metadata and take asset URLs out of the response, so a +/// base URL cannot stand in for either. Naming one would promise +/// something pnpm cannot do, and a closed set says so where the +/// configuration is written rather than after it is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Tool { + Node, + Bun, + Python, +} + +/// What pnpm is told about one tool it downloads, keyed by the tool +/// under `tools`. +/// +/// A tool here is a program pnpm fetches to run something with: a +/// JavaScript runtime, a Python interpreter, another package manager. +/// Where the packages of an ecosystem come from is a separate question, +/// answered by `registry`, `python.indexUrl` and `cargo.indexUrl`. +/// +/// Read from the global `config.yaml` and `PNPM_CONFIG_TOOLS` only. A +/// `pnpm-workspace.yaml` that names one is ignored, because naming a +/// mirror chooses which program runs on the machine of everyone who +/// clones the repository. +#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default, deny_unknown_fields)] +pub struct ToolSettings { + /// Where this tool's builds are downloaded from, in place of the + /// project that publishes them, as the base URL its own layout hangs + /// off. A tool that publishes several lines of builds has the line + /// below this base, the way its own tree lays them out. + pub mirror: Option, + /// Where one line of this tool's builds comes from, for a tool that + /// publishes more than one and a line that does not come from the + /// same place as the rest. An entry here answers for the channel it + /// names; every other channel is left to [`Self::mirror`]. + /// + /// Only Node.js publishes channels of the tools pnpm downloads, so + /// naming them for another is refused rather than left to do + /// nothing. [`Tool::has_channels`] is what decides. + pub channels: Option>, +} + +impl Tool { + /// Whether this tool publishes more than one line of builds. + #[must_use] + pub fn has_channels(self) -> bool { + match self { + Self::Node => true, + Self::Bun | Self::Python => false, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] #[serde(rename_all = "camelCase", default, deny_unknown_fields)] pub struct CargoSettings { @@ -136,11 +194,6 @@ pub struct PythonSettings { /// version such as `3.12` or a full one such as `3.12.7`. Empty locks /// for the version of the interpreter the install runs on. pub python_versions: Vec, - /// Where the interpreters pnpm installs are downloaded from, as the - /// releases URL of a [python-build-standalone] mirror. - /// - /// [python-build-standalone]: https://github.com/astral-sh/python-build-standalone - pub download_url: String, } impl Default for PythonSettings { @@ -155,7 +208,6 @@ impl Default for PythonSettings { extras: Vec::new(), groups: vec!["dev".to_string()], python_versions: Vec::new(), - download_url: DEFAULT_PYTHON_DOWNLOAD_URL.to_string(), } } } @@ -485,3 +537,24 @@ pub struct PeerDependencyMeta { #[serde(skip_serializing_if = "Option::is_none")] pub optional: Option, } + +/// Refuse `channels` for a tool that publishes one line of builds, where +/// it would otherwise sit in the configuration doing nothing. +/// +/// Checked as the value is read so that every source is covered: the +/// workspace file, the global `config.yaml` and `PNPM_CONFIG_TOOLS` all +/// arrive through serde. +pub(crate) fn deserialize_tools<'de, Deser: Deserializer<'de>>( + deserializer: Deser, +) -> Result>, Deser::Error> { + use serde::de::Error as _; + let tools = Option::>::deserialize(deserializer)?; + for (tool, settings) in tools.iter().flatten() { + if settings.channels.is_some() && !tool.has_channels() { + return Err(Deser::Error::custom(format!( + "{tool:?} publishes one line of builds, so it has no channels to name", + ))); + } + } + Ok(tools) +} diff --git a/pnpm/crates/config/src/workspace_yaml/settings.rs b/pnpm/crates/config/src/workspace_yaml/settings.rs index 398c110e91..53ebbd3083 100644 --- a/pnpm/crates/config/src/workspace_yaml/settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/settings.rs @@ -7,8 +7,9 @@ use super::{ PeerDependencyRules, Pipe, PmOnFail, PnpmfileSetting, PythonSettings, RegistryEntry, RemoteSideEffectsCacheSettings, ResolutionMode, RuntimeOnFail, SCHEMA_DIRECTIVE_KEY, SaveWorkspaceProtocol, ScriptsPrependNodePath, SideEffectsCacheSetting, SupportedArchitectures, - TaskSettings, TrustPolicy, UpdateConfig, UpdateSettings, VerifyDepsBeforeRun, VirtualStoreType, - WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, fs, redact_and_sanitize, + TaskSettings, Tool, ToolSettings, TrustPolicy, UpdateConfig, UpdateSettings, + VerifyDepsBeforeRun, VirtualStoreType, WORKSPACE_MANIFEST_FILENAME, WorkspaceKeyIssues, fs, + redact_and_sanitize, }; /// `serde` helper for fields that need to distinguish "missing key" @@ -165,6 +166,10 @@ pub struct WorkspaceSettings { pub pnpr_server: Option, pub cargo: Option, pub python: Option, + /// `tools` from `pnpm-workspace.yaml`: what pnpm is told about the + /// programs it downloads, keyed by tool name. See [`ToolSettings`]. + #[serde(default, deserialize_with = "crate::workspace_yaml::deserialize_tools")] + pub tools: Option>, pub remote_side_effects_cache: Option, pub https_proxy: Option, pub http_proxy: Option, diff --git a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs index 2fa5c06b60..ecfa36d4c5 100644 --- a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs @@ -312,9 +312,120 @@ cargo: assert!(settings.cargo.is_none()); } +#[test] +fn tool_settings_parse_and_apply() { + let yaml = "tools:\n node:\n mirror: https://mirror.example.test/node/download\n python:\n mirror: https://mirror.example.test/python-build-standalone/releases\n bun:\n mirror: https://mirror.example.test/bun\n"; + let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + let mut config = Config::default(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!( + config.tool_mirror(crate::Tool::Node), + Some("https://mirror.example.test/node/download"), + ); + assert_eq!( + config.tool_mirror(crate::Tool::Python), + Some("https://mirror.example.test/python-build-standalone/releases"), + ); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://mirror.example.test/bun")); + + // A mirror says what this machine can reach, which is the user's to + // say, so it survives the filter the global config is read through. + // The workspace layer drops it; `a_repository_cannot_name_a_mirror` + // covers that end. + let mut settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + settings.clear_workspace_only_fields(); + assert!(settings.tools.is_some()); + let unknown = + serde_saphyr::from_str::("tools:\n node:\n unknown: true\n"); + assert!(unknown.is_err()); +} + +/// Deno and Yarn are not reachable through a base URL at all, so the +/// setting cannot name one: the closed key set says so where the value +/// is written, rather than leaving an entry that does nothing. +#[test] +fn refuses_a_tool_pnpm_does_not_download() { + let error = serde_saphyr::from_str::( + "tools:\n deno:\n mirror: https://mirror.example.test/deno\n", + ) + .expect_err("deno is not a tool pnpm downloads through a mirror"); + assert!(format!("{error}").contains("deno"), "{error}"); + + let known = serde_saphyr::from_str::( + "tools:\n node:\n mirror: https://mirror.example.test/node\n", + ); + assert!(known.is_ok()); +} + +/// Only Node.js publishes more than one line of builds, so naming +/// channels for another would sit in the file doing nothing. +#[test] +fn refuses_channels_for_a_tool_that_has_none() { + let error = serde_saphyr::from_str::( + "tools:\n bun:\n channels:\n canary: https://mirror.example.test/bun\n", + ) + .expect_err("bun publishes one line of builds"); + assert!(format!("{error}").contains("channels"), "{error}"); + + let node = serde_saphyr::from_str::( + "tools:\n node:\n channels:\n nightly: https://mirror.example.test/node\n", + ); + assert!(node.is_ok()); +} + +/// Each tool is answered for separately, so naming one leaves the +/// mirrors the machine's own config names for the others alone. +#[test] +fn a_workspace_tool_keeps_the_mirrors_named_elsewhere() { + let mut config = Config::default(); + let global: WorkspaceSettings = + serde_saphyr::from_str("tools:\n node:\n mirror: https://global.example.test/node\n") + .unwrap(); + global.apply_to(&mut config, Path::new("/workspace")); + let workspace: WorkspaceSettings = + serde_saphyr::from_str("tools:\n bun:\n mirror: https://workspace.example.test/bun\n") + .unwrap(); + workspace.apply_to(&mut config, Path::new("/workspace")); + + assert_eq!(config.tool_mirror(crate::Tool::Node), Some("https://global.example.test/node")); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://workspace.example.test/bun")); +} + +#[test] +fn a_tool_channel_is_read_beside_the_base_it_refines() { + let yaml = "tools:\n node:\n mirror: https://mirror.example.test/node/download\n channels:\n nightly: https://nightly.example.test/\n"; + let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + let mut config = Config::default(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!( + config.tool_mirror(crate::Tool::Node), + Some("https://mirror.example.test/node/download"), + ); + assert_eq!( + config + .tool_channel_mirrors(crate::Tool::Node) + .get("nightly") + .map(String::as_str), + Some("https://nightly.example.test"), + ); + assert!(!config.tool_channel_mirrors(crate::Tool::Node).contains_key("release")); + assert!(config.tool_channel_mirrors(crate::Tool::Bun).is_empty()); +} + +/// A caller joins a path onto what it is given. +#[test] +fn a_tool_mirror_is_read_without_its_trailing_slash() { + let mut config = Config::default(); + let settings: WorkspaceSettings = + serde_saphyr::from_str("tools:\n bun:\n mirror: https://mirror.example.test/bun/\n") + .unwrap(); + settings.apply_to(&mut config, Path::new("/workspace")); + assert_eq!(config.tool_mirror(crate::Tool::Bun), Some("https://mirror.example.test/bun")); +} + #[test] fn python_settings_parse_apply_and_remain_workspace_only() { - let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n overrides: [demo>=2]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n pythonVersions: ['3.12', '3.13']\n downloadUrl: https://mirror.example.test/releases\n"; + let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n overrides: [demo>=2]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n pythonVersions: ['3.12', '3.13']\n"; let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); let mut config = Config::default(); settings.apply_to(&mut config, Path::new("/workspace")); @@ -327,7 +438,6 @@ fn python_settings_parse_apply_and_remain_workspace_only() { assert_eq!(config.python.extras, ["speed"]); assert_eq!(config.python.groups, ["test"]); assert_eq!(config.python.python_versions, ["3.12", "3.13"]); - assert_eq!(config.python.download_url, "https://mirror.example.test/releases"); let mut settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); settings.clear_workspace_only_fields(); assert!(settings.python.is_none()); diff --git a/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs b/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs index 80fa76a237..6f65eaa593 100644 --- a/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs +++ b/pnpm/crates/config/src/workspace_yaml/workspace_scope.rs @@ -52,6 +52,11 @@ impl WorkspaceSettings { self.clear_workspace_hooks_fields(); } + /// `tools` is deliberately absent: a mirror answers "what can this + /// machine reach", which is the user's to say and belongs in their + /// global config, the same way `nodeDownloadMirrors` does. Clearing + /// it here would leave a repository as the only party able to say + /// where pnpm downloads a runtime from. pub(super) fn clear_workspace_project_fields(&mut self) { self.versioning = None; self.cargo = None; diff --git a/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs b/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs index 4fb02ba2fa..68885171f6 100644 --- a/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs +++ b/pnpm/crates/deps-restorer/src/install_package_from_registry/tests.rs @@ -31,6 +31,7 @@ fn create_config( cache_dir: &Path, ) -> Config { Config { + tools: std::collections::BTreeMap::new(), bail: true, ci: false, update_notifier: true, diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs b/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs index 244631e579..4417d261be 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/bun_resolver.rs @@ -39,11 +39,21 @@ pub enum BunResolverError { pub struct BunResolver { pub http_client: Arc, pub npm_resolver: Arc, + /// `tools.bun.mirror`: where Bun's releases are downloaded from. + pub mirror: Option, } impl BunResolver { pub fn new(http_client: Arc, npm_resolver: Arc) -> Self { - Self { http_client, npm_resolver } + Self { http_client, npm_resolver, mirror: None } + } + + /// Download Bun's releases from `tools.bun.mirror` rather than from + /// Bun's own. + #[must_use] + pub fn with_mirror(mut self, mirror: Option<&str>) -> Self { + self.mirror = mirror.map(ToString::to_string); + self } } @@ -91,7 +101,8 @@ impl BunResolver { as ResolveError })?; - let variants = read_bun_assets(&self.http_client, &version).await + let variants = read_bun_assets(&self.http_client, self.mirror.as_deref(), &version) + .await .map_err(|err| Box::new(BunResolverError::ReadAssets(err)) as ResolveError)?; let resolution = LockfileResolution::Variations(VariationsResolution { variants }); let manifest = serde_json::json!({ diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs index 85bde6da04..b25619a8ed 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets.rs @@ -37,25 +37,36 @@ pub enum ReadBunAssetsError { /// Fetch and decode the Bun-release SHASUMS file for `version`. pub async fn read_bun_assets( http_client: &ThrottledClient, + mirror: Option<&str>, version: &str, ) -> Result, ReadBunAssetsError> { - let integrities_url = - format!("https://github.com/oven-sh/bun/releases/download/bun-v{version}/SHASUMS256.txt"); + let release = release_base(mirror, version); + let integrities_url = format!("{release}/SHASUMS256.txt"); let items = fetch_shasums_file(http_client, &integrities_url).await .map_err(ReadBunAssetsError::FetchShasumsFile)?; let mut variants = Vec::new(); for item in items { let Some(parsed) = parse_asset_name(&item.file_name) else { continue }; - variants.push(asset_resolution(version, &item, parsed)?); + variants.push(asset_resolution(&release, &item, parsed)?); } variants.sort_by(|a, b| variant_url(a).cmp(variant_url(b))); Ok(variants) } +/// Where one release's files are, under the mirror `tools.bun.mirror` +/// names or under Bun's own releases otherwise. Both lay a release out +/// the same way, so only the host above it differs. +fn release_base(mirror: Option<&str>, version: &str) -> String { + let base = mirror.map_or("https://github.com/oven-sh/bun/releases/download", |mirror| { + mirror.trim_end_matches('/') + }); + format!("{base}/bun-v{version}") +} + /// The download one `SHASUMS256.txt` entry describes. fn asset_resolution( - version: &str, + release: &str, item: &ShasumsFileItem, parsed: BunAssetName, ) -> Result { @@ -67,10 +78,7 @@ fn asset_resolution( error: Arc::new(error), })?; let binary = BinaryResolution { - url: format!( - "https://github.com/oven-sh/bun/releases/download/bun-v{version}/{file_name}", - file_name = item.file_name, - ), + url: format!("{release}/{file_name}", file_name = item.file_name), integrity, bin: BinarySpec::Single(bun_bin_path(&parsed.platform).to_string()), archive: BinaryArchive::Zip, diff --git a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs index e4bf884cf4..19917d8b37 100644 --- a/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs +++ b/pnpm/crates/engine-runtime-bun-resolver/src/read_bun_assets/tests.rs @@ -1,6 +1,6 @@ use pretty_assertions::assert_eq; -use super::parse_asset_name; +use super::{parse_asset_name, release_base}; #[test] fn parses_apple_silicon_zip() { @@ -32,3 +32,21 @@ fn ignores_unrelated_assets() { assert!(parse_asset_name("bun-linux.zip").is_none()); assert!(parse_asset_name("bun-linux-x64.tar.gz").is_none()); } + +/// A release's checksums and its assets share one base, so both move +/// with the host. +#[test] +fn a_release_is_laid_out_the_same_under_a_mirror() { + assert_eq!( + release_base(None, "1.2.3"), + "https://github.com/oven-sh/bun/releases/download/bun-v1.2.3", + ); + assert_eq!( + release_base(Some("https://mirror.example.test/bun"), "1.2.3"), + "https://mirror.example.test/bun/bun-v1.2.3", + ); + assert_eq!( + release_base(Some("https://mirror.example.test/bun/"), "1.2.3"), + "https://mirror.example.test/bun/bun-v1.2.3", + ); +} diff --git a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs index 58585316e1..5e21334f85 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror.rs @@ -11,18 +11,31 @@ pub const UNOFFICIAL_NODE_MIRROR_BASE_URL: &str = /// Resolve the base URL for a given release channel. /// -/// `node_download_mirrors` is the user's `.npmrc`/config override map -/// keyed by channel (`release`, `nightly`, `rc`, `test`, `v8-canary`). -/// A missing entry falls back to the official nodejs.org tree. The -/// returned URL always ends with `/` so callers can concatenate -/// `v/...` without a defensive check. +/// The three settings that can name it are read most specific first, +/// and the canonical spelling ahead of the older one where both are as +/// specific: +/// +/// 1. `channel`, from `tools.node.channels.`, which names this +/// channel and no other. +/// 2. `node_download_mirrors`, the older `node-mirror:` +/// spelling, kept because it has shipped. +/// 3. `mirror`, from `tools.node.mirror`, the base every channel hangs +/// off the way nodejs.org lays its own tree out. +/// +/// A channel none of them names falls back to the official nodejs.org +/// tree. The returned URL always ends with `/` so callers can +/// concatenate `v/...` without a defensive check. #[must_use] pub fn get_node_mirror( + mirror: Option<&str>, + channel: Option<&str>, node_download_mirrors: Option<&HashMap>, release_channel: &str, ) -> String { - let mirror = node_download_mirrors - .and_then(|map| map.get(release_channel).cloned()) + let mirror = channel + .map(ToString::to_string) + .or_else(|| node_download_mirrors.and_then(|map| map.get(release_channel).cloned())) + .or_else(|| mirror.map(|base| format!("{}/{release_channel}", base.trim_end_matches('/')))) .unwrap_or_else(|| format!("https://nodejs.org/download/{release_channel}/")); normalize_node_mirror(&mirror) } diff --git a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs index b437e32289..aeb1db76cf 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/get_node_mirror/tests.rs @@ -14,20 +14,85 @@ fn configured_mirror_per_channel_wins_over_default() { ("v8-canary", "http://test.mirror.localhost/v8-canary"), ] { let mirrors = HashMap::from([(channel.to_string(), host.to_string())]); - assert_eq!(get_node_mirror(Some(&mirrors), channel), format!("{host}/")); + assert_eq!(get_node_mirror(None, None, Some(&mirrors), channel), format!("{host}/")); } } #[test] fn uses_defaults_when_unconfigured() { let empty = HashMap::new(); - assert_eq!(get_node_mirror(Some(&empty), "release"), "https://nodejs.org/download/release/"); - assert_eq!(get_node_mirror(None, "release"), "https://nodejs.org/download/release/"); + assert_eq!( + get_node_mirror(None, None, Some(&empty), "release"), + "https://nodejs.org/download/release/", + ); + assert_eq!( + get_node_mirror(None, None, None, "release"), + "https://nodejs.org/download/release/", + ); } #[test] fn appends_trailing_slash_when_missing() { let mirrors = HashMap::from([("release".to_string(), "http://test.mirror.localhost".to_string())]); - assert_eq!(get_node_mirror(Some(&mirrors), "release"), "http://test.mirror.localhost/"); + assert_eq!( + get_node_mirror(None, None, Some(&mirrors), "release"), + "http://test.mirror.localhost/", + ); +} + +/// nodejs.org lays its channels out below one base, which is the shape +/// a mirror of it carries. +#[test] +fn the_tool_mirror_is_the_base_every_channel_hangs_off() { + for channel in ["release", "nightly", "v8-canary"] { + assert_eq!( + get_node_mirror(Some("http://test.mirror.localhost/download"), None, None, channel), + format!("http://test.mirror.localhost/download/{channel}/"), + ); + } + assert_eq!( + get_node_mirror(Some("http://test.mirror.localhost/download/"), None, None, "release"), + "http://test.mirror.localhost/download/release/", + ); +} + +#[test] +fn a_channel_named_outright_wins_over_the_base() { + let mirrors = HashMap::from([("nightly".to_string(), "http://nightly.localhost".to_string())]); + assert_eq!( + get_node_mirror(Some("http://base.localhost"), None, Some(&mirrors), "nightly"), + "http://nightly.localhost/", + ); + assert_eq!( + get_node_mirror(Some("http://base.localhost"), None, Some(&mirrors), "release"), + "http://base.localhost/release/", + ); +} + +#[test] +fn a_channel_overrides_the_base_it_is_named_beside() { + assert_eq!( + get_node_mirror( + Some("http://base.localhost/download"), + Some("http://nightly.localhost"), + None, + "nightly", + ), + "http://nightly.localhost/", + ); + assert_eq!( + get_node_mirror(Some("http://base.localhost/download"), None, None, "release"), + "http://base.localhost/download/release/", + ); +} + +/// `node-mirror:` is the older spelling of the same thing. +#[test] +fn the_canonical_channel_wins_over_the_older_spelling() { + let mirrors = HashMap::from([("nightly".to_string(), "http://older.localhost".to_string())]); + assert_eq!( + get_node_mirror(None, Some("http://canonical.localhost"), Some(&mirrors), "nightly"), + "http://canonical.localhost/", + ); } diff --git a/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs b/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs index 213687e435..594fcb68c1 100644 --- a/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs +++ b/pnpm/crates/engine-runtime-node-resolver/src/node_resolver.rs @@ -97,6 +97,11 @@ pub struct NodeResolver { pub http_client: Arc, pub auth_headers: Arc, pub node_download_mirrors: HashMap, + /// `tools.node.mirror`: the base every release channel hangs off. + pub mirror: Option, + /// `tools.node.channels`: where one release channel comes from when + /// it does not come from the same place as the rest. + pub channel_mirrors: HashMap, pub offline: bool, /// The pnpm cache directory backing the per-version SHASUMS disk /// cache. `None` disables the cache and every resolve fetches the @@ -119,6 +124,8 @@ impl NodeResolver { http_client, auth_headers, node_download_mirrors: HashMap::new(), + mirror: None, + channel_mirrors: HashMap::new(), offline: false, cache_dir: None, } @@ -240,7 +247,12 @@ impl NodeResolver { } let parsed = parse_node_specifier(version_spec).map_err(NodeResolverError::InvalidReleaseChannel)?; - let mirror = get_node_mirror(Some(&self.node_download_mirrors), &parsed.release_channel); + let mirror = get_node_mirror( + self.mirror.as_deref(), + self.channel_mirrors.get(&parsed.release_channel).map(String::as_str), + Some(&self.node_download_mirrors), + &parsed.release_channel, + ); if let Some(version) = exact_release_version(&parsed) { return Ok(PickedNodeVersion { version, @@ -312,7 +324,12 @@ impl NodeResolver { .map_err(|err| { Box::new(NodeResolverError::InvalidReleaseChannel(err)) as ResolveError })?; - let mirror = get_node_mirror(Some(&self.node_download_mirrors), &parsed.release_channel); + let mirror = get_node_mirror( + self.mirror.as_deref(), + self.channel_mirrors.get(&parsed.release_channel).map(String::as_str), + Some(&self.node_download_mirrors), + &parsed.release_channel, + ); let version = resolve_node_version_with_auth( &self.http_client, &self.auth_headers, diff --git a/pnpm/crates/package-manager/src/add/specifier.rs b/pnpm/crates/package-manager/src/add/specifier.rs index 3f4cc3e779..cb817188d0 100644 --- a/pnpm/crates/package-manager/src/add/specifier.rs +++ b/pnpm/crates/package-manager/src/add/specifier.rs @@ -6,7 +6,7 @@ use super::{ }; use crate::{CatalogModeDep, decide_catalog_outcome}; use pnpm_catalogs_types::Catalogs; -use pnpm_config::{Config, SaveWorkspaceProtocol}; +use pnpm_config::{Config, SaveWorkspaceProtocol, Tool}; use pnpm_engine_runtime_node_resolver::NodeResolver; use pnpm_package_manifest::{DependencyGroup, PackageManifest}; use pnpm_package_name::is_valid_dependency_alias; @@ -199,6 +199,8 @@ pub(super) async fn resolve_node_runtime_specifier( std::sync::Arc::clone(&config.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&config.node_download_mirrors); + node_resolver.mirror = config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = config.tool_channel_mirrors(Tool::Node); node_resolver.offline = config.offline; node_resolver.cache_dir = Some(config.cache_dir.clone()); node_resolver diff --git a/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs b/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs index 4aec877a6f..6e6f52bf15 100644 --- a/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs +++ b/pnpm/crates/package-manager/src/install_with_fresh_lockfile/resolver_setup.rs @@ -8,7 +8,7 @@ use super::InstallWithFreshLockfileError; use crate::{PrefetchContext, PrefetchingResolver}; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -339,6 +339,8 @@ impl ResolverChainInputs<'_> { Arc::clone(self.fetching.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&self.config.node_download_mirrors); + node_resolver.mirror = self.config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = self.config.tool_channel_mirrors(Tool::Node); node_resolver.offline = self.config.offline; node_resolver.cache_dir = Some(self.config.cache_dir.clone()); node_resolver @@ -407,10 +409,10 @@ impl ResolverChainInputs<'_> { Arc::clone(self.fetching.http_client), Arc::clone(npm_resolver), )), - Box::new(BunResolver::new( - Arc::clone(self.fetching.http_client), - Arc::clone(npm_resolver), - )), + Box::new( + BunResolver::new(Arc::clone(self.fetching.http_client), Arc::clone(npm_resolver)) + .with_mirror(self.config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new( Arc::clone(self.fetching.http_client), self.config.tls.strict_ssl.unwrap_or(true), diff --git a/pnpm/crates/package-manager/src/update/latest.rs b/pnpm/crates/package-manager/src/update/latest.rs index 4ab7b5a4c8..f692aeb395 100644 --- a/pnpm/crates/package-manager/src/update/latest.rs +++ b/pnpm/crates/package-manager/src/update/latest.rs @@ -8,7 +8,7 @@ use crate::{ }; use chrono::{DateTime, Utc}; use node_semver::Version; -use pnpm_config::{Config, version_policy::PackageVersionPolicy}; +use pnpm_config::{Config, Tool, version_policy::PackageVersionPolicy}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -180,13 +180,18 @@ pub(super) fn ensure_latest_resolver_chain<'chain>( Arc::clone(&ctx.config.auth_headers), ); node_resolver.node_download_mirrors.clone_from(&ctx.config.node_download_mirrors); + node_resolver.mirror = ctx.config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = ctx.config.tool_channel_mirrors(Tool::Node); node_resolver.offline = ctx.config.offline; node_resolver.cache_dir = Some(ctx.config.cache_dir.clone()); let resolver = DefaultResolver::new(vec![ Box::new(Arc::clone(&npm_resolver)) as Box, Box::new(node_resolver), Box::new(DenoResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver))), - Box::new(BunResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver))), + Box::new( + BunResolver::new(Arc::clone(ctx.http_client_arc), Arc::clone(&npm_resolver)) + .with_mirror(ctx.config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new( Arc::clone(ctx.http_client_arc), ctx.config.tls.strict_ssl.unwrap_or(true), diff --git a/pnpm/crates/python-installer/src/interpreter/download.rs b/pnpm/crates/python-installer/src/interpreter/download.rs index a38cb3d71d..9cd181327e 100644 --- a/pnpm/crates/python-installer/src/interpreter/download.rs +++ b/pnpm/crates/python-installer/src/interpreter/download.rs @@ -9,7 +9,7 @@ use super::{InterpreterCommand, VersionRequest, command::interpreter_in}; use miette::{IntoDiagnostic, Result, WrapErr, bail}; -use pnpm_config::{Config, RuntimeOnFail}; +use pnpm_config::{Config, DEFAULT_PYTHON_DOWNLOAD_URL, RuntimeOnFail, Tool}; use pnpm_crypto_shasums_file::{ShasumsFileItem, fetch_moving_shasums_file_cached}; use pnpm_network::{AuthHeaders, ThrottledClient}; use pnpm_reporter::{GlobalLog, LogEvent, LogLevel, Reporter}; @@ -56,10 +56,18 @@ pub(super) struct Build { integrity: ssri::Integrity, } +/// Where the interpreter builds are downloaded from: the mirror +/// `tools.python.mirror` names, or python-build-standalone's own +/// releases. A mirror lays a release out the way that project does, so +/// only the host above it differs. +fn releases_url(config: &Config) -> &str { + config.tool_mirror(Tool::Python).unwrap_or(DEFAULT_PYTHON_DOWNLOAD_URL) +} + impl Releases { /// The interpreters pnpm can install. pub(super) async fn read(config: &Config, client: &ThrottledClient) -> Result { - let url = format!("{}/latest/download/SHA256SUMS", config.python.download_url); + let url = format!("{}/latest/download/SHA256SUMS", releases_url(config)); let index = fetch_moving_shasums_file_cached( client, &url, @@ -129,7 +137,7 @@ impl Build { config: &Config, client: &ThrottledClient, ) -> Result { - let url = format!("{}/download/{}/{}", config.python.download_url, self.tag, self.file); + let url = format!("{}/download/{}/{}", releases_url(config), self.tag, self.file); let response = client .get_limited_bytes_with_secure_auth_and_retry( &url, diff --git a/pnpm/crates/resolving-default-resolver/src/standalone.rs b/pnpm/crates/resolving-default-resolver/src/standalone.rs index 748d660422..4419c2bb35 100644 --- a/pnpm/crates/resolving-default-resolver/src/standalone.rs +++ b/pnpm/crates/resolving-default-resolver/src/standalone.rs @@ -20,7 +20,7 @@ //! install-time concern. use crate::DefaultResolver; -use pnpm_config::Config; +use pnpm_config::{Config, Tool}; use pnpm_engine_pm_yarn_resolver::YarnResolver; use pnpm_engine_runtime_bun_resolver::BunResolver; use pnpm_engine_runtime_deno_resolver::DenoResolver; @@ -84,7 +84,10 @@ pub fn build_standalone_chain( Box::new(LocalSchemeResolver::new(local_ctx)), Box::new(build_node_resolver(config, http_client)), Box::new(DenoResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver))), - Box::new(BunResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver))), + Box::new( + BunResolver::new(Arc::clone(http_client), Arc::clone(&npm_resolver)) + .with_mirror(config.tool_mirror(Tool::Bun)), + ), Box::new(YarnResolver::new(Arc::clone(http_client), config.tls.strict_ssl.unwrap_or(true))), Box::new(build_named_registry_resolver(opts, retry_opts)?), Box::new(LocalPathResolver::new(local_ctx)), @@ -148,6 +151,8 @@ fn build_node_resolver(config: &Config, http_client: &Arc) -> N let mut node_resolver = NodeResolver::new_with_auth(Arc::clone(http_client), Arc::clone(&config.auth_headers)); node_resolver.node_download_mirrors.clone_from(&config.node_download_mirrors); + node_resolver.mirror = config.tool_mirror(Tool::Node).map(ToString::to_string); + node_resolver.channel_mirrors = config.tool_channel_mirrors(Tool::Node); node_resolver.offline = config.offline; node_resolver.cache_dir = Some(config.cache_dir.clone()); node_resolver