diff --git a/.changeset/python-ecosystem-overrides.md b/.changeset/python-ecosystem-overrides.md new file mode 100644 index 0000000000..75c481eb08 --- /dev/null +++ b/.changeset/python-ecosystem-overrides.md @@ -0,0 +1,5 @@ +--- +"pacquet": patch +--- + +The shared `overrides` setting now accepts a `pypi` subfield for Python dependency rules. diff --git a/pnpm/crates/cli/tests/suite/python.rs b/pnpm/crates/cli/tests/suite/python.rs index cb7eba8828..04d7b3ffdb 100644 --- a/pnpm/crates/cli/tests/suite/python.rs +++ b/pnpm/crates/cli/tests/suite/python.rs @@ -179,14 +179,25 @@ async fn serve_wheels( fn add_python_settings(root: &Path, settings: &str) { let workspace = fs::read_to_string(root.join("pnpm-workspace.yaml")).unwrap(); - fs::write( - root.join("pnpm-workspace.yaml"), - workspace.replace( - "python:\n enabled: true\n", - &format!("python:\n enabled: true\n{settings}"), - ), - ) - .unwrap(); + let mut python_settings = String::new(); + let mut pypi_overrides = None; + for line in settings.lines() { + if let Some(value) = line.strip_prefix(" overrides: ") { + pypi_overrides = Some(value); + } else { + writeln!(python_settings, "{line}").unwrap(); + } + } + let workspace = workspace.replace( + "python:\n enabled: true\n", + &format!("python:\n enabled: true\n{python_settings}"), + ); + let workspace = if let Some(overrides) = pypi_overrides { + format!("{workspace}\noverrides:\n pypi: {overrides}\n") + } else { + workspace + }; + fs::write(root.join("pnpm-workspace.yaml"), workspace).unwrap(); } /// The platform of the machine running the tests, as `python.platforms` diff --git a/pnpm/crates/config/src/workspace_yaml.rs b/pnpm/crates/config/src/workspace_yaml.rs index 655498a5c1..12ec92b960 100644 --- a/pnpm/crates/config/src/workspace_yaml.rs +++ b/pnpm/crates/config/src/workspace_yaml.rs @@ -2,10 +2,10 @@ pub mod package_configs; pub mod registries; pub use error::LoadWorkspaceYamlError; pub use sections::{ - AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, PackageExtension, - PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, PythonDownloads, PythonSettings, - RemoteSideEffectsCacheSettings, SideEffectsCacheSetting, SideEffectsCacheSettings, - TaskSettings, UpdateConfig, UpdateSettings, decided_allow_builds, + AllowBuild, AuditSettings, CargoSettings, DEFAULT_PYTHON_DOWNLOAD_URL, EcosystemOverrides, + OverridesSetting, PackageExtension, PeerDependencyMeta, PeerDependencyRules, PnpmfileSetting, + PythonDownloads, PythonSettings, RemoteSideEffectsCacheSettings, SideEffectsCacheSetting, + SideEffectsCacheSettings, TaskSettings, UpdateConfig, UpdateSettings, decided_allow_builds, }; pub use settings::WorkspaceSettings; diff --git a/pnpm/crates/config/src/workspace_yaml/apply.rs b/pnpm/crates/config/src/workspace_yaml/apply.rs index 8da5a9072c..8d06463862 100644 --- a/pnpm/crates/config/src/workspace_yaml/apply.rs +++ b/pnpm/crates/config/src/workspace_yaml/apply.rs @@ -163,7 +163,11 @@ impl WorkspaceSettings { overlay_some(&mut config.scope, self.scope.take()); overlay_some(&mut config.pnpr_server, self.pnpr_server.take()); overlay(&mut config.cargo, self.cargo.take()); - overlay(&mut config.python, self.python.take()); + if let Some(python) = self.python.take() { + let overrides = config.python.overrides.clone(); + config.python = python; + config.python.overrides = overrides; + } if let Some(v) = self.remote_side_effects_cache.take() { config.remote_side_effects_cache.get_or_insert_default().overlay(v); } @@ -294,7 +298,9 @@ impl WorkspaceSettings { // once the cascade knows the workspace root, whose manifest // carries the direct dependencies they point at. if let Some(v) = self.overrides.take() { - config.overrides = (!v.is_empty()).then_some(v); + let npm = v.npm(); + config.overrides = (!npm.is_empty()).then_some(npm); + config.python.overrides = v.pypi().to_vec(); } if let Some(v) = self.package_extensions.take() { config.package_extensions = (!v.is_empty()).then_some(v); diff --git a/pnpm/crates/config/src/workspace_yaml/reset.rs b/pnpm/crates/config/src/workspace_yaml/reset.rs index 35d05e0eb8..758714f7c8 100644 --- a/pnpm/crates/config/src/workspace_yaml/reset.rs +++ b/pnpm/crates/config/src/workspace_yaml/reset.rs @@ -4,6 +4,12 @@ use super::{ }; type Reset = fn(&mut Config, &Config); + +fn reset_overrides(config: &mut Config, defaults: &Config) { + config.overrides.clone_from(&defaults.overrides); + config.python.overrides.clone_from(&defaults.python.overrides); +} + fn apply_named_reset( config: &mut Config, defaults: &Config, @@ -128,7 +134,7 @@ impl WorkspaceSettings { ignore_scripts, ignore_pnpmfile, git_checks, engine_strict, node_version, runtime_on_fail, node_download_mirrors, scripts_prepend_node_path, script_shell, node_options, unsafe_perm, supported_architectures, ignored_optional_dependencies, - overrides, package_extensions, package_configs, minimum_release_age_exclude, + package_extensions, package_configs, minimum_release_age_exclude, minimum_release_age_ignore_missing_time, minimum_release_age_strict, trust_lockfile, trust_policy, trust_policy_exclude, trust_policy_exclude_prune, trust_policy_ignore_after, init_author_name, init_author_email, init_author_url, @@ -136,6 +142,7 @@ impl WorkspaceSettings { save_prefix, pipeline_base, child_concurrency, workspace_concurrency, catalogs, allow_builds, concurrency_groups, }); + resets.push(("overrides".to_string(), reset_overrides as Reset)); if Self::reset_derived_setting_to_default::(config, defaults, key, base_dir) { return true; } diff --git a/pnpm/crates/config/src/workspace_yaml/resolved.rs b/pnpm/crates/config/src/workspace_yaml/resolved.rs index a9ab69f043..c0d88063e2 100644 --- a/pnpm/crates/config/src/workspace_yaml/resolved.rs +++ b/pnpm/crates/config/src/workspace_yaml/resolved.rs @@ -1,8 +1,22 @@ use super::{ - AllowBuild, Config, PackageConfigsSetting, PnpmfileSetting, WorkspaceSettings, as_set, - global_shims_setting, opt_path, path, side_effects_cache_setting, + AllowBuild, Config, EcosystemOverrides, OverridesSetting, PackageConfigsSetting, + PnpmfileSetting, WorkspaceSettings, as_set, global_shims_setting, opt_path, path, + side_effects_cache_setting, }; +fn resolved_overrides(config: &Config) -> Option { + let npm = config.overrides.clone().unwrap_or_default(); + let pypi = config.python.overrides.clone(); + if npm.is_empty() && pypi.is_empty() { + return None; + } + if pypi.is_empty() { + Some(OverridesSetting::Legacy(npm)) + } else { + Some(OverridesSetting::Ecosystems(EcosystemOverrides { npm, pypi })) + } +} + impl WorkspaceSettings { /// Every setting at the value `config` resolved it to, for a consumer /// that must read the effective configuration rather than one file's @@ -135,7 +149,7 @@ impl WorkspaceSettings { unsafe_perm: Some(config.unsafe_perm), supported_architectures: config.supported_architectures.clone(), ignored_optional_dependencies: config.ignored_optional_dependencies.clone(), - overrides: config.overrides.clone(), + overrides: resolved_overrides(config), package_extensions: config.package_extensions.clone(), // The flattened lookup, which is the by-name form of the setting // whichever of the two forms the file wrote it in. diff --git a/pnpm/crates/config/src/workspace_yaml/sections.rs b/pnpm/crates/config/src/workspace_yaml/sections.rs index 5bd0ba72f2..05addbddf4 100644 --- a/pnpm/crates/config/src/workspace_yaml/sections.rs +++ b/pnpm/crates/config/src/workspace_yaml/sections.rs @@ -15,6 +15,57 @@ pub enum AllowBuild { Undecided(String), } +/// Ecosystem-specific dependency overrides. The legacy flat map remains +/// accepted as the npm form. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] +#[serde(untagged)] +pub enum OverridesSetting { + Legacy(IndexMap), + Ecosystems(EcosystemOverrides), +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub struct EcosystemOverrides { + #[serde(default)] + pub npm: IndexMap, + #[serde(default)] + pub pypi: Vec, +} + +impl OverridesSetting { + pub fn iter(&self) -> Box + '_> { + match self { + Self::Legacy(overrides) => Box::new(overrides.iter()), + Self::Ecosystems(overrides) => Box::new(overrides.npm.iter()), + } + } + + #[must_use] + pub fn is_empty(&self) -> bool { + match self { + Self::Legacy(overrides) => overrides.is_empty(), + Self::Ecosystems(overrides) => { + overrides.npm.is_empty() && overrides.pypi.is_empty() + } + } + } + + pub fn npm(&self) -> IndexMap { + match self { + Self::Legacy(overrides) => overrides.clone(), + Self::Ecosystems(overrides) => overrides.npm.clone(), + } + } + + pub fn pypi(&self) -> &[String] { + match self { + Self::Legacy(_) => &[], + Self::Ecosystems(overrides) => &overrides.pypi, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, Deserialize)] #[serde(untagged)] pub enum PnpmfileSetting { @@ -123,12 +174,13 @@ impl Default for CargoSettings { )] pub struct PythonSettings { pub enabled: bool, + #[serde(skip)] + pub overrides: Vec, /// The interpreter to install every Python project with. `None` lets /// pnpm choose one the project accepts. pub executable: Option, pub index_url: String, pub extra_index_urls: Vec, - pub overrides: Vec, pub constraints: Vec, pub extras: Vec, pub groups: Vec, @@ -165,10 +217,10 @@ impl Default for PythonSettings { fn default() -> Self { Self { enabled: false, + overrides: Vec::new(), executable: None, index_url: "https://pypi.org/simple/".to_string(), extra_index_urls: Vec::new(), - overrides: Vec::new(), constraints: Vec::new(), extras: Vec::new(), groups: vec!["dev".to_string()], diff --git a/pnpm/crates/config/src/workspace_yaml/settings.rs b/pnpm/crates/config/src/workspace_yaml/settings.rs index 398c110e91..e04193e42a 100644 --- a/pnpm/crates/config/src/workspace_yaml/settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/settings.rs @@ -3,7 +3,7 @@ use super::{ CatalogMode, ConfigDependency, Deserialize, Deserializer, DroppedKeys, ErrorKind, GLOBAL_CONFIG_YAML_FILENAME, HashMap, HoistingLimits, IgnoredAny, IndexMap, InitType, LinkWorkspacePackages, LoadWorkspaceYamlError, NodeLinker, NodePackageMapType, - PackageConfigsSetting, PackageExtension, PackageImportMethod, Path, PathBuf, + OverridesSetting, PackageConfigsSetting, PackageExtension, PackageImportMethod, Path, PathBuf, PeerDependencyRules, Pipe, PmOnFail, PnpmfileSetting, PythonSettings, RegistryEntry, RemoteSideEffectsCacheSettings, ResolutionMode, RuntimeOnFail, SCHEMA_DIRECTIVE_KEY, SaveWorkspaceProtocol, ScriptsPrependNodePath, SideEffectsCacheSetting, SupportedArchitectures, @@ -512,7 +512,7 @@ pub struct WorkspaceSettings { /// `pnpm_lockfile::check_lockfile_settings` compares this /// against `lockfile.overrides` and raises `OverridesChanged` /// on mismatch. - pub overrides: Option>, + pub overrides: Option, /// `cacheDir` from `pnpm-workspace.yaml`. Resolved against the /// workspace dir like the other path-valued fields. Drives diff --git a/pnpm/crates/config/src/workspace_yaml/tests/behavior_color_accepts_boolean_compatibility.rs b/pnpm/crates/config/src/workspace_yaml/tests/behavior_color_accepts_boolean_compatibility.rs index 95af3d945e..2337fbbb94 100644 --- a/pnpm/crates/config/src/workspace_yaml/tests/behavior_color_accepts_boolean_compatibility.rs +++ b/pnpm/crates/config/src/workspace_yaml/tests/behavior_color_accepts_boolean_compatibility.rs @@ -2,6 +2,7 @@ use super::{ AllowBuild, ColorMode, Config, Path, RegistryEntry, SideEffectsCacheSetting, WORKSPACE_MANIFEST_FILENAME, WorkspaceSettings, assert_eq, fs, }; +use crate::workspace_yaml::OverridesSetting; #[test] fn color_accepts_boolean_compatibility_values() { @@ -741,6 +742,52 @@ overrides: assert_eq!(applied.get("baz>qux").map(String::as_str), Some("-")); } +#[test] +fn parses_ecosystem_overrides_and_applies_python_rules() { + let settings: WorkspaceSettings = serde_saphyr::from_str( + "overrides:\n npm:\n foo: '1.2.3'\n pypi:\n - 'requests>=2,<3'\n", + ) + .unwrap(); + let mut config = Config::new(); + settings.apply_to(&mut config, Path::new("/irrelevant")); + assert_eq!( + config.overrides + .unwrap() + .get("foo") + .map(String::as_str), + Some("1.2.3"), + ); + assert_eq!(config.python.overrides, ["requests>=2,<3"]); +} + +#[test] +fn python_settings_layer_preserves_shared_pypi_overrides() { + let mut config = Config::new(); + serde_saphyr::from_str::("overrides:\n pypi:\n - 'requests>=2,<3'\n") + .unwrap() + .apply_to(&mut config, Path::new("/irrelevant")); + serde_saphyr::from_str::("python:\n enabled: true\n") + .unwrap() + .apply_to(&mut config, Path::new("/irrelevant")); + assert_eq!(config.python.overrides, ["requests>=2,<3"]); +} + +#[test] +fn resolved_settings_include_pypi_overrides() { + let mut config = Config::new(); + serde_saphyr::from_str::("overrides:\n pypi:\n - 'requests>=2,<3'\n") + .unwrap() + .apply_to(&mut config, Path::new("/irrelevant")); + let resolved = WorkspaceSettings::from_resolved(&config); + assert_eq!( + resolved.overrides + .as_ref() + .unwrap() + .pypi(), + ["requests>=2,<3"], + ); +} + /// An empty `overrides:` map collapses to `None` on `Config`, matching /// upstream's `delete settings.overrides` short-circuit in /// `getOptionsFromPnpmSettings`. Without this collapse, an empty @@ -750,7 +797,7 @@ overrides: fn empty_overrides_map_collapses_to_none() { let yaml = "overrides: {}\n"; let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); - assert!(settings.overrides.as_ref().is_some_and(indexmap::IndexMap::is_empty)); + assert!(settings.overrides.as_ref().is_some_and(OverridesSetting::is_empty)); let mut config = Config::new(); settings.apply_to(&mut config, Path::new("/irrelevant")); diff --git a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs index dd312240a6..b194da0fc5 100644 --- a/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs +++ b/pnpm/crates/config/src/workspace_yaml/tests/workspace_settings.rs @@ -314,7 +314,7 @@ cargo: #[test] fn python_settings_parse_apply_and_remain_workspace_only() { - let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n overrides: [demo>=2]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n platforms: [x86_64-manylinux_2_28, aarch64-apple-darwin]\n pythonVersions: ['3.12', '3.13']\n downloads: never\n downloadUrl: https://mirror.example.test/releases\n"; + let yaml = "python:\n enabled: true\n executable: python3.13\n indexUrl: https://example.org/simple/\n extraIndexUrls: [https://extra.example.org/simple/]\n constraints: [demo<3]\n extras: [speed]\n groups: [test]\n platforms: [x86_64-manylinux_2_28, aarch64-apple-darwin]\n pythonVersions: ['3.12', '3.13']\n downloads: never\n downloadUrl: https://mirror.example.test/releases\n"; let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); let mut config = Config::default(); settings.apply_to(&mut config, Path::new("/workspace")); @@ -322,7 +322,6 @@ fn python_settings_parse_apply_and_remain_workspace_only() { assert_eq!(config.python.executable.as_deref(), Some("python3.13")); assert_eq!(config.python.index_url, "https://example.org/simple/"); assert_eq!(config.python.extra_index_urls, ["https://extra.example.org/simple/"]); - assert_eq!(config.python.overrides, ["demo>=2"]); assert_eq!(config.python.constraints, ["demo<3"]); assert_eq!(config.python.extras, ["speed"]); assert_eq!(config.python.groups, ["test"]); diff --git a/pnpm/plans/PYTHON_SPIKE.md b/pnpm/plans/PYTHON_SPIKE.md index 266fe83dfd..ff3f1aad37 100644 --- a/pnpm/plans/PYTHON_SPIKE.md +++ b/pnpm/plans/PYTHON_SPIKE.md @@ -40,13 +40,15 @@ A credential fingerprint separates authenticated index caches, and raw credentials never appear in cache keys. Missing index pages are cached for offline resolution too. -`python.overrides` and `python.constraints` accept lists of PEP 508 registry +`overrides.pypi` and `python.constraints` accept lists of PEP 508 registry requirements. An override replaces the version requirement for a matching name throughout the dependency graph, including its requested extras. Constraints intersect the permitted versions without adding a dependency. Markers select where a rule applies. URL requirements are not accepted in these lists. Version rules preserve the source of dependencies declared as Git repositories or direct wheel URLs. +The shared `overrides` setting has an `npm` map and a `pypi` list. The legacy +flat map remains the npm form. For example: ```yaml