diff --git a/rust/exo_rs/exo_rs.pyi b/rust/exo_rs/exo_rs.pyi index b95d1956a..b06635828 100644 --- a/rust/exo_rs/exo_rs.pyi +++ b/rust/exo_rs/exo_rs.pyi @@ -158,10 +158,16 @@ class BootstrapArgs: class BootstrapSettings: @property def exo_home(self) -> ExoHome: ... + @exo_home.setter + def exo_home(self, value: ExoHome) -> None: ... @property def models_dirs(self) -> ModelsDirs: ... + @models_dirs.setter + def models_dirs(self, value: ModelsDirs) -> None: ... @property def log_files(self) -> LogFiles: ... + @log_files.setter + def log_files(self, value: LogFiles) -> None: ... @property def pid_file(self) -> pathlib.Path: ... @pid_file.setter @@ -202,9 +208,6 @@ class BootstrapSettings: """ @staticmethod def resolve(args: BootstrapArgs) -> BootstrapSettings: ... - def set_exo_home(self, exo_home: ExoHome) -> None: ... - def set_models_dirs(self, models_dirs: ModelsDirs) -> None: ... - def set_log_files(self, log_files: LogFiles) -> None: ... def to_bytes(self) -> builtins.list[builtins.int]: ... @staticmethod def from_bytes(bytes: typing.Sequence[builtins.int]) -> BootstrapSettings: ... @@ -250,10 +253,16 @@ class CliArgs: def discovery_port(self, value: builtins.int) -> None: ... @property def bootstrap(self) -> BootstrapArgs: ... + @bootstrap.setter + def bootstrap(self, value: BootstrapArgs) -> None: ... @property def app(self) -> AppArgs: ... + @app.setter + def app(self, value: AppArgs) -> None: ... @property def rejected(self) -> RejectedArgs: ... + @rejected.setter + def rejected(self, value: RejectedArgs) -> None: ... @staticmethod def from_env_only() -> CliArgs: r""" @@ -263,9 +272,6 @@ class CliArgs: def parse_from(argv: typing.Sequence[builtins.str]) -> CliArgs: ... @staticmethod def parse() -> CliArgs: ... - def set_bootstrap(self, bootstrap: BootstrapArgs) -> None: ... - def set_app(self, app: AppArgs) -> None: ... - def set_rejected(self, rejected: RejectedArgs) -> None: ... def to_bytes(self) -> builtins.list[builtins.int]: ... @staticmethod def from_bytes(bytes: typing.Sequence[builtins.int]) -> CliArgs: ... diff --git a/rust/exo_rs/src/bin/stub_gen.rs b/rust/exo_rs/src/bin/stub_gen.rs index 8d57b0b8e..2990155bc 100644 --- a/rust/exo_rs/src/bin/stub_gen.rs +++ b/rust/exo_rs/src/bin/stub_gen.rs @@ -2,15 +2,21 @@ use clap::Parser; use exo_rs::config::app::AppSettings; use exo_rs::config::bootstrap::BootstrapSettings; use exo_rs::config::cli::CliArgs; +use pyo3::Python; use pyo3_stub_gen::Result; fn main() -> Result<()> { - let a = CliArgs::parse(); - println!("{a:?}\n"); - let b = BootstrapSettings::resolve(&a.bootstrap)?; - println!("{b:?}\n"); - let app = AppSettings::resolve(&a.app, &b)?; - println!("{app:?}\n"); + Python::initialize(); + + Python::attach(|py| -> Result<()> { + let a = CliArgs::parse(); + println!("{a:?}\n"); + let b = BootstrapSettings::resolve(py, &a.bootstrap.borrow(py))?; + println!("{b:?}\n"); + let app = AppSettings::resolve(&a.app.borrow(py), &b)?; + println!("{app:?}\n"); + Ok(()) + })?; // return Ok(()); diff --git a/rust/exo_rs/src/config/app.rs b/rust/exo_rs/src/config/app.rs index 037b4bbe5..c1b7daeeb 100644 --- a/rust/exo_rs/src/config/app.rs +++ b/rust/exo_rs/src/config/app.rs @@ -11,7 +11,7 @@ use figment::Figment; use figment::providers::{Format, Serialized, Toml}; use pyo3::prelude::{PyModule, PyModuleMethods}; use pyo3::types::PyTuple; -use pyo3::{Bound, PyAny, PyResult, pyclass, pymethods}; +use pyo3::{Bound, PyAny, PyResult, Python, pyclass, pymethods}; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; @@ -164,18 +164,18 @@ impl AppSettings { /// Create default instance. #[staticmethod] #[pyo3(name = "default")] - pub fn py_default() -> PyResult { - let bootstrap = BootstrapSettings::py_default()?; + pub fn py_default(py: Python<'_>) -> PyResult { + let bootstrap = BootstrapSettings::py_default(py)?; let args = AppArgs::default(); Self::resolve(&args, &bootstrap) } /// Create only from environment variables. #[staticmethod] - pub fn from_env_only() -> PyResult { + pub fn from_env_only(py: Python<'_>) -> PyResult { let args = CliArgs::from_env_only(); - let bootstrap = BootstrapSettings::resolve(&args.bootstrap)?; - Self::resolve(&args.app, &bootstrap) + let bootstrap = BootstrapSettings::resolve(py, &*args.bootstrap.borrow(py))?; + Self::resolve(&*args.app.borrow(py), &bootstrap) } #[staticmethod] diff --git a/rust/exo_rs/src/config/bootstrap.rs b/rust/exo_rs/src/config/bootstrap.rs index a745cae9c..2856bb5d2 100644 --- a/rust/exo_rs/src/config/bootstrap.rs +++ b/rust/exo_rs/src/config/bootstrap.rs @@ -1,10 +1,11 @@ use crate::config::cli::CliArgs; use crate::config::cli::{PathBufValueParserExt, parse_path}; use crate::ext::ResultExt; +use crate::newtype::NewPy; use crate::pickle_reduce; use pyo3::prelude::{PyModule, PyModuleMethods}; use pyo3::types::PyTuple; -use pyo3::{Bound, PyAny, PyResult, pyclass, pymethods}; +use pyo3::{Bound, PyAny, PyResult, Python, pyclass, pymethods}; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use serde::{Deserialize, Serialize}; use std::io; @@ -79,17 +80,17 @@ pub struct BootstrapArgs { } #[gen_stub_pyclass] -#[pyclass(module = "exo_rs", from_py_object)] +#[pyclass(module = "exo_rs", skip_from_py_object)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct BootstrapSettings { - #[pyo3(get)] - pub exo_home: ExoHome, + #[pyo3(get, set)] + pub exo_home: NewPy, - #[pyo3(get)] - pub models_dirs: ModelsDirs, + #[pyo3(get, set)] + pub models_dirs: NewPy, - #[pyo3(get)] - pub log_files: LogFiles, + #[pyo3(get, set)] + pub log_files: NewPy, // other #[pyo3(get, set)] @@ -120,20 +121,20 @@ impl BootstrapSettings { /// Create default instance #[staticmethod] #[pyo3(name = "default")] - pub fn py_default() -> PyResult { + pub fn py_default(py: Python<'_>) -> PyResult { // resolve from env only - Self::resolve(&BootstrapArgs::default()) + Self::resolve(py, &BootstrapArgs::default()) } /// Create only from env-variables #[staticmethod] - pub fn from_env_only() -> PyResult { + pub fn from_env_only(py: Python<'_>) -> PyResult { // resolve from env only - Self::resolve(&CliArgs::from_env_only().bootstrap) + Self::resolve(py, &*CliArgs::from_env_only().bootstrap.borrow(py)) } #[staticmethod] - pub fn resolve(args: &BootstrapArgs) -> PyResult { + pub fn resolve(py: Python<'_>, args: &BootstrapArgs) -> PyResult { let exo_home = ExoHome::resolve(args)?; let models_dirs = ModelsDirs::resolve(args, &exo_home)?; let log_files = LogFiles::resolve(&exo_home)?; @@ -156,9 +157,9 @@ impl BootstrapSettings { let tracing_cache_dir = exo_home.cache.join("traces"); Ok(Self { - exo_home, - models_dirs, - log_files, + exo_home: NewPy::py_try_new_with(py, exo_home)?, + models_dirs: NewPy::py_try_new_with(py, models_dirs)?, + log_files: NewPy::py_try_new_with(py, log_files)?, pid_file, node_zid, config_file, @@ -169,18 +170,6 @@ impl BootstrapSettings { }) } - pub fn set_exo_home(&mut self, exo_home: ExoHome) { - self.exo_home = exo_home; - } - - pub fn set_models_dirs(&mut self, models_dirs: ModelsDirs) { - self.models_dirs = models_dirs; - } - - pub fn set_log_files(&mut self, log_files: LogFiles) { - self.log_files = log_files; - } - // -------- SERDE/PICKLING support -------- pub fn to_bytes(&self) -> PyResult> { @@ -198,7 +187,7 @@ impl BootstrapSettings { } #[gen_stub_pyclass] -#[pyclass(from_py_object)] +#[pyclass(skip_from_py_object)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ExoHome { #[pyo3(get, set)] @@ -250,7 +239,7 @@ impl ExoHome { } #[gen_stub_pyclass] -#[pyclass(from_py_object)] +#[pyclass(skip_from_py_object)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ModelsDirs { #[pyo3(get, set)] @@ -290,7 +279,7 @@ impl ModelsDirs { } #[gen_stub_pyclass] -#[pyclass(from_py_object)] +#[pyclass(skip_from_py_object)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct LogFiles { #[pyo3(get, set)] diff --git a/rust/exo_rs/src/config/cli.rs b/rust/exo_rs/src/config/cli.rs index 59c30e7da..e9d270f8e 100644 --- a/rust/exo_rs/src/config/cli.rs +++ b/rust/exo_rs/src/config/cli.rs @@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize}; use std::ffi::OsString; // re-export +use crate::newtype::NewPy; pub use parser_impl::*; #[gen_stub_pyclass] @@ -102,16 +103,16 @@ pub struct CliArgs { // -------- FLATTENED SUBCOMMANDS -------- #[command(flatten)] - #[pyo3(get)] - pub bootstrap: BootstrapArgs, + #[pyo3(get, set)] + pub bootstrap: NewPy, #[command(flatten)] - #[pyo3(get)] - pub app: AppArgs, + #[pyo3(get, set)] + pub app: NewPy, #[command(flatten)] - #[pyo3(get)] - pub rejected: RejectedArgs, + #[pyo3(get, set)] + pub rejected: NewPy, } #[gen_stub_pymethods] @@ -139,18 +140,6 @@ impl CliArgs { Ok(CliArgs::parse_from(argv)) } - pub fn set_bootstrap(&mut self, bootstrap: BootstrapArgs) { - self.bootstrap = bootstrap; - } - - pub fn set_app(&mut self, app: AppArgs) { - self.app = app; - } - - pub fn set_rejected(&mut self, rejected: RejectedArgs) { - self.rejected = rejected; - } - // -------- SERDE/PICKLING support -------- pub fn to_bytes(&self) -> PyResult> { @@ -205,6 +194,7 @@ pub struct RejectedArgs { pub libp2p_port: Option, #[arg( + long = Rejected::fake_name(), env = "EXO_LIBP2P_NAMESPACE", value_name = "STRING", hide = true, value_parser = Rejected::::deprecated( None, None, Some("EXO_LIBP2P_NAMESPACE"), @@ -215,6 +205,7 @@ pub struct RejectedArgs { pub libp2p_namespace: Option, #[arg( + long = Rejected::fake_name(), env = "EXO_ZENOH_NAMESPACE", value_name = "STRING", hide = true, value_parser = Rejected::::deprecated( None, None, Some("EXO_ZENOH_NAMESPACE"), @@ -225,6 +216,7 @@ pub struct RejectedArgs { pub zenoh_namespace: Option, #[arg( + long = Rejected::fake_name(), env = "EXO_ENABLE_IMAGE_MODELS", value_name = "BOOL", hide = true, value_parser = Rejected::::deprecated( None, None, Some("EXO_ENABLE_IMAGE_MODELS"), @@ -235,6 +227,7 @@ pub struct RejectedArgs { pub enable_image_models: Option, #[arg( + long = Rejected::fake_name(), env = "ENABLE_DISAGGREGATION", value_name = "BOOL", hide = true, value_parser = Rejected::::deprecated( None, None, Some("ENABLE_DISAGGREGATION"), @@ -271,6 +264,7 @@ mod parser_impl { use clap::builder::PathBufValueParser; use clap::builder::TypedValueParser; use itertools::Itertools; + use rand::distr::SampleString; use std::error::Error; use std::ffi::OsStr; use std::fs; @@ -280,11 +274,17 @@ mod parser_impl { use util::path::{PathExt, resolve_path}; #[derive(Clone)] - pub struct Rejected { + pub struct Rejected { message: String, _ty: PhantomData, } + impl Rejected { + pub fn fake_name() -> String { + rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 32) + } + } + impl Rejected { #[inline(always)] pub fn new(message: impl Into) -> Self { diff --git a/rust/util/src/path.rs b/rust/util/src/path.rs index d45526ca1..110ece157 100644 --- a/rust/util/src/path.rs +++ b/rust/util/src/path.rs @@ -161,7 +161,9 @@ pub fn resolve_path(path: PathBuf) -> io::Result { ); // 5) prefix/suffix joined in final absolute clean path with symlinks resolved and no ".." - prefix.push(&suffix); + if !suffix.as_os_str().is_empty() { + prefix.push(&suffix); + } let path = prefix; assert!( path.components() @@ -268,4 +270,20 @@ mod tests { assert_eq!(resolved, real_parent.canonicalize().unwrap()); assert_is_root_followed_by_normal_components(&resolved); } + + #[test] + fn resolve_path_supports_existing_file() { + let test_dir = TempDir::new().unwrap(); + let file = test_dir.path().join("config.toml"); + fs::write(&file, "verbosity = \"warn\"\n").unwrap(); + + let resolved = resolve_path(file.clone()).unwrap(); + + assert_eq!(resolved, file.canonicalize().unwrap()); + assert_eq!( + resolved.canonicalize().unwrap(), + file.canonicalize().unwrap() + ); + assert_is_root_followed_by_normal_components(&resolved); + } } diff --git a/src/exo/download/tests/test_download_verification.py b/src/exo/download/tests/test_download_verification.py index 698758ef8..a6e514091 100644 --- a/src/exo/download/tests/test_download_verification.py +++ b/src/exo/download/tests/test_download_verification.py @@ -27,10 +27,8 @@ def model_id() -> ModelId: def _mock_bootstrap_settings(models_dir: Path) -> BootstrapSettings: cfg = BootstrapSettings.default() - models_dirs = cfg.models_dirs - models_dirs.default_models_dir = models_dir - models_dirs.models_dirs = [models_dir] - cfg.set_models_dirs(models_dirs) + cfg.models_dirs.default_models_dir = models_dir + cfg.models_dirs.models_dirs = [models_dir] return cfg diff --git a/src/exo/download/tests/test_model_dirs.py b/src/exo/download/tests/test_model_dirs.py index 7daf1d434..baa4bb13f 100644 --- a/src/exo/download/tests/test_model_dirs.py +++ b/src/exo/download/tests/test_model_dirs.py @@ -51,11 +51,9 @@ def _mock_bootstrap_settings( read_only: list[Path] | None = None, ) -> BootstrapSettings: cfg = BootstrapSettings.default() - models_dirs = cfg.models_dirs - models_dirs.default_models_dir = default - models_dirs.models_dirs = writable or [] - models_dirs.models_read_only_dirs = read_only or [] - cfg.set_models_dirs(models_dirs) + cfg.models_dirs.default_models_dir = default + cfg.models_dirs.models_dirs = writable or [] + cfg.models_dirs.models_read_only_dirs = read_only or [] return cfg diff --git a/src/exo/download/tests/test_offline_mode.py b/src/exo/download/tests/test_offline_mode.py index 2bfbb5aca..0ddb25634 100644 --- a/src/exo/download/tests/test_offline_mode.py +++ b/src/exo/download/tests/test_offline_mode.py @@ -27,10 +27,8 @@ def model_id() -> ModelId: def _mock_bootstrap_settings(models_dir: Path) -> BootstrapSettings: cfg = BootstrapSettings.default() - models_dirs = cfg.models_dirs - models_dirs.default_models_dir = models_dir - models_dirs.models_dirs = [models_dir] - cfg.set_models_dirs(models_dirs) + cfg.models_dirs.default_models_dir = models_dir + cfg.models_dirs.models_dirs = [models_dir] return cfg