diff --git a/Cargo.lock b/Cargo.lock index 0649c80b4..3be38be3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" @@ -182,7 +182,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -194,7 +194,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -216,7 +216,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -227,7 +227,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -392,9 +392,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -628,7 +628,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -639,7 +639,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -701,7 +701,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -745,7 +745,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -801,7 +801,7 @@ dependencies = [ ] [[package]] -name = "exo_pyo3_bindings" +name = "exo_net" version = "0.0.1" dependencies = [ "env_logger", @@ -809,6 +809,7 @@ dependencies = [ "futures-lite", "log", "networking", + "parking_lot", "pidfile-rs", "pin-project", "pyo3", @@ -819,7 +820,6 @@ dependencies = [ "serde_json", "tokio", "zenoh", - "zerompk", ] [[package]] @@ -830,7 +830,7 @@ checksum = "311a6d2f1f9d60bff73d2c78a0af97ed27f79672f15c238192a5bbb64db56d00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1006,7 +1006,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1116,7 +1116,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1155,6 +1155,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heck" version = "0.5.0" @@ -1364,12 +1370,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1394,9 +1400,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -1419,7 +1425,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1482,7 +1488,7 @@ checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1528,7 +1534,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1556,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -1852,11 +1858,9 @@ dependencies = [ "parking_lot", "rand 0.10.1", "tokio", - "tracing", "zenoh", "zenoh-plugin-storage-manager", "zenoh-plugin-trait", - "zerompk", ] [[package]] @@ -2090,9 +2094,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "5.1.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -2193,7 +2197,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2214,7 +2218,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.14.0", "serde", ] @@ -2278,7 +2282,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2327,7 +2331,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2441,7 +2445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2455,9 +2459,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -2503,7 +2507,7 @@ checksum = "bcd7d70ee0ca1661c40407e6f84e4463ef2658c90a9e2fbbd4515b2bcdfcaeca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2545,7 +2549,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2558,19 +2562,19 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] name = "pyo3-stub-gen" -version = "0.17.2" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398b833826a83ca72c1e26d1b2c7c71f9ca7c3bfc74eacc663901895c362ae33" +checksum = "b695173a33bec37b6acd288efe74f902f123f5891e5ff5ad53ff429e18833c14" dependencies = [ "anyhow", "chrono", "either", - "indexmap 2.12.1", + "indexmap 2.14.0", "inventory", "itertools 0.14.0", "log", @@ -2580,22 +2584,25 @@ dependencies = [ "ordered-float", "pyo3", "pyo3-stub-gen-derive", + "rustpython-parser", "serde", - "toml", + "serde_json", + "time", + "toml 1.1.2+spec-1.1.0", ] [[package]] name = "pyo3-stub-gen-derive" -version = "0.17.2" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2426ba759d848787239d80f9fdb1f223786976f87fb6c3da8188ca7c17744b28" +checksum = "c2f901834d55c74f36be3353994062e3c3b3d47fda25572f5535e99434612ac8" dependencies = [ "heck", - "indexmap 2.12.1", + "indexmap 2.14.0", "proc-macro2", "quote", "rustpython-parser", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -2657,9 +2664,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -2809,7 +2816,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3135,7 +3142,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3210,7 +3217,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3221,7 +3228,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3239,9 +3246,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] @@ -3256,7 +3263,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -3274,7 +3281,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3283,7 +3290,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -3528,9 +3535,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -3545,7 +3552,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3580,7 +3587,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3591,7 +3598,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3717,7 +3724,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -3762,13 +3769,28 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.14.0", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.3", "toml_parser", "toml_writer", - "winnow", + "winnow 0.7.14", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.2", ] [[package]] @@ -3780,32 +3802,41 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" dependencies = [ - "indexmap 2.12.1", - "toml_datetime", + "indexmap 2.14.0", + "toml_datetime 0.7.3", "toml_parser", - "winnow", + "winnow 0.7.14", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.2", ] [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tracing" @@ -3827,7 +3858,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4060,7 +4091,7 @@ checksum = "3b5bb2756c16fb66f80cfbf5fb0e0c09a7001e739f453c9ec241b9c8b1556fda" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4135,7 +4166,7 @@ checksum = "8c44ce98e7227a04eeb4cf9c784109a5c9710e54849ceb4f09f8597247897f1e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "unzip-n", ] @@ -4229,7 +4260,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4259,7 +4290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.12.1", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -4272,7 +4303,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.14.0", "semver", ] @@ -4388,7 +4419,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4399,7 +4430,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -4676,6 +4707,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4710,9 +4747,9 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.12.1", + "indexmap 2.14.0", "prettyplease", - "syn 2.0.111", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4728,7 +4765,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -4741,7 +4778,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap 2.12.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -4760,7 +4797,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.12.1", + "indexmap 2.14.0", "log", "semver", "serde", @@ -4828,7 +4865,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -4927,7 +4964,7 @@ dependencies = [ "serde_json", "serde_with", "serde_yaml", - "toml", + "toml 0.9.8", "tracing", "uhlc", "validated_struct", @@ -5190,7 +5227,7 @@ checksum = "9310b02a8f6dc4bd04d9ce6b318b9d00182aeeeeca60410003307d63a2569a3f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "zenoh-keyexpr", ] @@ -5406,7 +5443,7 @@ checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] @@ -5426,7 +5463,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", "synstructure", ] @@ -5436,26 +5473,6 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" -[[package]] -name = "zerompk" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4bfaccde2671513aa564585b2e9ccf0a0ccaf9689477abfd60e53d3806afa51" -dependencies = [ - "zerompk_derive", -] - -[[package]] -name = "zerompk_derive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2197ce2b0c4f95ebfd8b09134ecfe0dcc152481b21bacd7a4b1976df1ccfdfc8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "zerotrie" version = "0.2.3" @@ -5486,7 +5503,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7e85ac663..64b39dbb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["rust/exo_pyo3_bindings", "rust/networking"] +members = ["rust/exo_net", "rust/networking"] [workspace.package] version = "0.0.1" @@ -44,6 +44,7 @@ pin-project = "1.1.10" serde_json = "1.0.149" rand = "0.10.1" parking_lot = "0.12.5" +pidfile-rs = "0.3.1" # Tracing/logging log = "0.4" diff --git a/flake.nix b/flake.nix index eaff877da..64533c6da 100644 --- a/flake.nix +++ b/flake.nix @@ -110,7 +110,7 @@ nixpkgs-fmt.enable = true; ruff-format = { enable = true; - excludes = [ "rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi" ]; + excludes = [ "rust/exo_net/exo_net.pyi" ]; }; rustfmt = { enable = true; diff --git a/justfile b/justfile index 6ef6fa4e0..b6ab068cf 100644 --- a/justfile +++ b/justfile @@ -23,7 +23,7 @@ sync-clean: rust-rebuild: PYO3_PYTHON="$(uv run python -c 'import sys; print(sys.executable)')" cargo run --bin stub_gen - uv sync --reinstall-package exo_pyo3_bindings + uv sync --reinstall-package exo_net build-dashboard: #!/usr/bin/env bash diff --git a/pyproject.toml b/pyproject.toml index 1ffe06b8a..e0d2e5907 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "huggingface-hub>=1.8.0", "psutil>=7.0.0", "loguru>=0.7.3", - "exo-pyo3-bindings", # rust bindings + "exo-net", # rust bindings "anyio==4.11.0", "tiktoken>=0.12.0", # required for kimi k2 tokenizer "hypercorn>=0.18.0", @@ -76,10 +76,10 @@ mlx-cuda13 = [ ### [tool.uv.workspace] -members = ["rust/exo_pyo3_bindings", "bench", "tools"] +members = ["rust/exo_net", "bench", "tools"] [tool.uv.sources] -exo-pyo3-bindings = { workspace = true } +exo-net = { workspace = true } mlx = [ { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }, { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx-0.32.0-cp313-cp313-manylinux_2_35_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" }, @@ -241,7 +241,7 @@ torchaudio = ["torch"] ### [tool.ruff] -extend-exclude = [".typings/**", "rust/exo_pyo3_bindings/**", "bench/vendor/**"] +extend-exclude = [".typings/**", "rust/exo_net/**", "bench/vendor/**"] [tool.ruff.lint] extend-select = ["I", "N", "B", "A", "PIE", "SIM"] diff --git a/python/parts.nix b/python/parts.nix index 18c01e062..9ccabff5f 100644 --- a/python/parts.nix +++ b/python/parts.nix @@ -47,17 +47,18 @@ let # Replace workspace exo_pyo3_bindings with Nix-built wheel. # Preserve passthru so mkVirtualEnv can resolve dependency groups. # Copy .pyi stub + py.typed marker so basedpyright can find the types. - exo-pyo3-bindings = pkgs.stdenv.mkDerivation { - pname = "exo-pyo3-bindings"; + exo-net = pkgs.stdenv.mkDerivation { + pname = "exo-net"; version = "0.1.0"; - src = self'.packages.exo_pyo3_bindings; + src = self'.packages.exo-net; # Install from pre-built wheel nativeBuildInputs = [ final.pyprojectWheelHook ]; dontStrip = true; passthru = prev.exo-pyo3-bindings.passthru or { }; + postInstall = '' - local siteDir=$out/${final.python.sitePackages}/exo_pyo3_bindings - cp ${inputs.self}/rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi $siteDir/ + local siteDir=$out/${final.python.sitePackages}/exo_net + cp ${inputs.self}/rust/exo_net/exo_net.pyi $siteDir/ touch $siteDir/py.typed ''; }; diff --git a/rust/exo_net/Cargo.toml b/rust/exo_net/Cargo.toml index 42487c225..da8ce09df 100644 --- a/rust/exo_net/Cargo.toml +++ b/rust/exo_net/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "exo_pyo3_bindings" +name = "exo_net" version = { workspace = true } edition = { workspace = true } publish = false @@ -7,7 +7,7 @@ publish = false [lib] doctest = false path = "src/lib.rs" -name = "exo_pyo3_bindings" +name = "exo_net" # "cdylib" needed to produce shared library for Python to import # "rlib" needed for stub-gen to run @@ -26,26 +26,14 @@ networking.workspace = true extend.workspace = true # interop -pyo3 = { version = "0.27.2", features = [ - # "abi3-py313", # tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.13 - # "nightly", # enables better-supported GIL integration - "experimental-async", # async support in #[pyfunction] & #[pymethods] - #"experimental-inspect", # inspection of generated binary => easier to automate type-hint generation - #"py-clone", # adding Clone-ing of `Py` without GIL (may cause panics - remove if panics happen) - # "multiple-pymethods", # allows multiple #[pymethods] sections per class - - # integrations with other libraries - # "arc_lock", "bigdecimal", "either", "hashbrown", "indexmap", "num-bigint", "num-complex", "num-rational", - # "ordered-float", "rust_decimal", "smallvec", - # "anyhow", "chrono", "chrono-local", "chrono-tz", "eyre", "jiff-02", "lock_api", "parking-lot", "time", "serde", -] } -pyo3-stub-gen = { version = "0.17.2" } -pyo3-async-runtimes = { version = "0.27.0", features = [ +pyo3 = { workspace = true, features = ["experimental-async"] } +pyo3-stub-gen.workspace = true +pyo3-async-runtimes = { workspace = true, features = [ "attributes", "tokio-runtime", "testing", ] } -pyo3-log = "0.13.2" +pyo3-log.workspace = true pidfile-rs = { git = "https://github.com/AndreiCravtov/pidfile-rs" } @@ -68,3 +56,4 @@ zenoh.workspace = true rand.workspace = true serde_json.workspace = true parking_lot.workspace = true +pidfile-rs.workspace = true diff --git a/rust/exo_net/exo_net.pyi b/rust/exo_net/exo_net.pyi new file mode 100644 index 000000000..0eacc5934 --- /dev/null +++ b/rust/exo_net/exo_net.pyi @@ -0,0 +1,94 @@ +# This file is automatically generated by pyo3_stub_gen +# ruff: noqa: E501, F401, F403, F405 + +import builtins +import os +import pathlib +import typing +__all__ = [ + "NetworkingHandle", + "Pidfile", + "PidfileError", + "PyFromSwarm", +] + +@typing.final +class NetworkingHandle: + @staticmethod + def new(identity: bytes, bootstrap_peers: typing.Sequence[builtins.str], listen_port: builtins.int) -> NetworkingHandle: ... + async def gossipsub_subscribe(self, topic: builtins.str) -> builtins.bool: + r""" + Subscribe to a `GossipSub` topic. + + Returns `True` if the subscription worked. Returns `False` if we were already subscribed. + """ + async def gossipsub_unsubscribe(self, topic: builtins.str) -> builtins.bool: + r""" + Unsubscribes from a `GossipSub` topic. + + Returns `True` if we were subscribed to this topic. Returns `False` if we were not subscribed. + """ + async def gossipsub_publish(self, topic: builtins.str, data: bytes) -> None: + r""" + Publishes a message with multiple topics to the `GossipSub` network. + + If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception. + """ + async def recv(self) -> PyFromSwarm: ... + +@typing.final +class Pidfile: + r""" + A PID file protected with a lock. + + An instance of `Pidfile` can be used to manage a PID file: create it, + lock it, detect already running daemons. It is backed by [`pidfile`][] + functions of `libbsd`/`libutil` which use `flopen` to lock the PID + file. + + When a PID file is created, the process ID of the current process is + *not* written there, making it possible to lock the PID file before + forking and only write the ID of the forked process when it is ready. + + The PID file is deleted automatically when the `Pidfile` comes out of + the scope. To close the PID file without deleting it, for example, in + the parent process of a forked daemon, call `close()`. + + [`exit`]: https://doc.rust-lang.org/std/process/fn.exit.html + [`pidfile`]: https://linux.die.net/man/3/pidfile + [`daemon`(3)]: https://linux.die.net/man/3/daemon + """ + def __new__(cls, path: builtins.str | os.PathLike | pathlib.Path, mode: builtins.int) -> Pidfile: + r""" + Creates a new PID file and locks it. + Writes the current process ID to the PID file. + + If the PID file cannot be locked, returns `PidfileError::AlreadyRunning` with + a PID of the already running process, or `None` if no PID has been written to + the PID file yet. + """ + +@typing.final +class PidfileError(builtins.Exception): + def __repr__(self) -> builtins.str: ... + def __str__(self) -> builtins.str: ... + +class PyFromSwarm: + @typing.final + class Connection(PyFromSwarm): + __match_args__ = ("connected",) + @property + def connected(self) -> builtins.bool: ... + def __new__(cls, connected: builtins.bool) -> PyFromSwarm.Connection: ... + + @typing.final + class Message(PyFromSwarm): + __match_args__ = ("topic", "data",) + @property + def topic(self) -> builtins.str: ... + @property + def data(self) -> bytes: ... + def __new__(cls, topic: builtins.str, data: bytes) -> PyFromSwarm.Message: ... + + ... + diff --git a/rust/exo_net/pyproject.toml b/rust/exo_net/pyproject.toml index fec8c1648..e7d8d359c 100644 --- a/rust/exo_net/pyproject.toml +++ b/rust/exo_net/pyproject.toml @@ -3,24 +3,22 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" [project] -name = "exo_pyo3_bindings" -version = "0.2.10" +name = "exo_net" +version = "0.3.0" description = "Add your description here" readme = "README.md" authors = [ - { name = "Andrei Cravtov", email = "the.andrei.cravtov@gmail.com" }, { name = "Evan Quiney", email = "evanev7@gmail.com" }, + { name = "Andrei Cravtov", email = "the.andrei.cravtov@gmail.com" }, ] requires-python = ">=3.13" dependencies = [] [dependency-groups] -dev = ["exo_pyo3_bindings", "pytest>=8.4.0", "pytest-asyncio>=1.0.0"] +dev = ["exo-net", "pytest>=8.4.0", "pytest-asyncio>=1.0.0"] [tool.maturin] -#purelib = true -#python-source = "python" -module-name = "exo_pyo3_bindings" +module-name = "exo_net" features = ["pyo3/extension-module", "pyo3/experimental-async"] [tool.pytest.ini_options] diff --git a/rust/exo_net/src/bin/stub_gen.rs b/rust/exo_net/src/bin/stub_gen.rs index 3e30f4939..bd9b4a53e 100644 --- a/rust/exo_net/src/bin/stub_gen.rs +++ b/rust/exo_net/src/bin/stub_gen.rs @@ -2,7 +2,7 @@ use pyo3_stub_gen::Result; fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().filter_or("RUST_LOG", "info")).init(); - let stub = exo_pyo3_bindings::stub_info()?; + let stub = exo_net::stub_info()?; stub.generate()?; Ok(()) } diff --git a/rust/exo_net/src/lib.rs b/rust/exo_net/src/lib.rs index 3d9ff0002..c7c8d6d38 100644 --- a/rust/exo_net/src/lib.rs +++ b/rust/exo_net/src/lib.rs @@ -7,11 +7,10 @@ mod allow_threading; mod pidfile; // mod ident; -// mod networking; -mod state; +mod networking; +use crate::networking::networking_submodule; use crate::pidfile::pidfile_submodule; -use crate::state::snapshot_module; use pyo3::prelude::PyModule; use pyo3::{Bound, PyResult, pymodule}; use pyo3_stub_gen::define_stub_info_gatherer; @@ -147,7 +146,7 @@ pub(crate) mod ext { /// A Python module implemented in Rust. The name of this function must match /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. -#[pymodule(name = "exo_pyo3_bindings")] +#[pymodule(name = "exo_net")] fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> { // install logger pyo3_log::init(); @@ -161,7 +160,7 @@ fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> { pidfile_submodule(m)?; // m.add_class::()?; // networking_submodule(m)?; - snapshot_module(m)?; + networking_submodule(m)?; // top-level constructs // TODO: ... diff --git a/rust/exo_net/src/networking.rs b/rust/exo_net/src/networking.rs index 62388c7d5..ab001101a 100644 --- a/rust/exo_net/src/networking.rs +++ b/rust/exo_net/src/networking.rs @@ -1,15 +1,12 @@ use std::pin::Pin; use std::sync::Arc; -use crate::r#const::MPSC_CHANNEL_SIZE; use crate::ext::{ByteArrayExt as _, FutureExt, PyErrExt as _}; use crate::ext::{ResultExt as _, TokioMpscSenderExt as _}; -use crate::ident::PyKeypair; -use crate::pyclass; use futures_lite::{Stream, StreamExt as _}; use networking::swarm::{FromSwarm, ToSwarm, create_swarm}; -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{PyModule, PyModuleMethods as _}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::{Bound, Py, PyAny, PyErr, PyResult, Python, pymethods}; use pyo3_stub_gen::derive::{ @@ -53,27 +50,35 @@ impl PyNetworkingHandle { // ---- Lifecycle management methods ---- - #[new] - #[pyo3(signature = (identity, bootstrap_peers, listen_port))] - fn py_new( - identity: Bound<'_, PyKeypair>, + #[staticmethod] + fn new<'py>( + identity: Bound<'py, PyBytes>, bootstrap_peers: Vec, listen_port: u16, - ) -> PyResult { + ) -> PyResult { // create communication channels - let (to_swarm, from_client) = mpsc::channel(MPSC_CHANNEL_SIZE); + let (to_swarm, from_client) = mpsc::channel(1024); // get identity - let identity = identity.borrow().0.clone(); + let identity = u128::from_le_bytes( + identity + .extract::<'_, Vec>()? + .try_into() + .map_err(|_| PyValueError::new_err("invalid identity bytes"))?, + ); // create networking swarm (within tokio context!! or it crashes) - let _guard = pyo3_async_runtimes::tokio::get_runtime().enter(); - let swarm = create_swarm(identity, from_client, bootstrap_peers, listen_port) - .map(|it| it.into_stream()) + let swarm = pyo3_async_runtimes::tokio::get_runtime() + .block_on(create_swarm( + identity, + from_client, + bootstrap_peers, + listen_port, + )) .pyerr()?; - Ok(Self { - swarm: Arc::new(Mutex::new(swarm)), + Ok(PyNetworkingHandle { + swarm: Arc::new(Mutex::new(swarm.into_stream())), to_swarm, }) } diff --git a/rust/exo_net/src/pidfile.rs b/rust/exo_net/src/pidfile.rs index 679ab1a06..722d8d918 100644 --- a/rust/exo_net/src/pidfile.rs +++ b/rust/exo_net/src/pidfile.rs @@ -77,6 +77,7 @@ impl PyPidfile { #[pymethods] impl PyPidfile { /// Creates a new PID file and locks it. + /// Writes the current process ID to the PID file. /// /// If the PID file cannot be locked, returns `PidfileError::AlreadyRunning` with /// a PID of the already running process, or `None` if no PID has been written to diff --git a/rust/exo_net/src/state.rs b/rust/exo_net/src/state.rs deleted file mode 100644 index dc73262e5..000000000 --- a/rust/exo_net/src/state.rs +++ /dev/null @@ -1,110 +0,0 @@ -use networking::Session; -use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, -}; -use pyo3_stub_gen::derive::{gen_methods_from_python, gen_stub_pyclass, gen_stub_pymethods}; -use serde_json::{Map, Value}; -use zenoh::{Result, Session as ZSession, query::QueryTarget, sample::SampleFields}; - -pyo3_stub_gen::inventory::submit! { - gen_methods_from_python! { - r#" - class StateProxy: - @staticmethod - async def init() -> StateProxy: ... - async def snapshot(self) -> str: ... - "# - } -} - -pub fn snapshot_module(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; - - Ok(()) -} - -#[gen_stub_pyclass] -#[pyclass] -pub struct StateProxy { - session: Session, -} -#[gen_stub_pymethods] -#[pymethods] -impl StateProxy { - #[staticmethod] - #[gen_stub(skip)] - pub fn init<'py>(py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(Self { - session: networking::open( - networking::cfg(rand::random(), 0).expect("default cfg is valid"), - ) - .await - .map_err(|e| PyRuntimeError::new_err(e.to_string()))?, - }) - }) - } - - #[gen_stub(skip)] - pub fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, { - let session = self.session.clone(); - async move { - Self::_snapshot(session) - .await - .map_err(|e| PyValueError::new_err(e.to_string())) - .map(|v| v.to_string()) - } - }) - } -} - -impl StateProxy { - async fn _snapshot(session: ZSession) -> Result { - let q = session.get("storage/mem1/**").await?; - - let mut v = Value::Object(Map::default()); - - while let Ok(sample) = q.recv_async().await { - let mut cur_v = &mut v; - let Ok(sample) = sample.into_result() else { - continue; - }; - // skip storage/mem1 - let SampleFields { - payload, key_expr, .. - } = sample.into(); - let mut iter = key_expr.split('/').skip(2).peekable(); - loop { - let Some(p) = iter.next() else { - break; - }; - if iter.peek().is_none() { - // terminal; write value into json - let existing = cur_v - .as_object_mut() - .expect("path terminated unexpectedly - value stored at some/path and some/path/two") - .insert(p.to_owned(), Value::String(payload.try_to_string()?.to_string())); - - if let Some(value) = existing { - assert!(value.is_string()) - // could log, but string overwrites are fine - } - } else { - // non-terminal; ensure key exists in v, then replace cur with that object - cur_v = cur_v - .as_object_mut() - .expect("path terminated unexpectedly - value stored at some/path and some/path/two") - .entry(p) - .or_insert(Value::Object(Map::default())); - assert!( - cur_v.is_object(), - "path terminated unexpectedly - value stored at some/path and some/path/two" - ) - } - } - } - Ok(v) - } -} diff --git a/rust/exo_net/tests/test_python.py b/rust/exo_net/tests/test_python.py index 742278703..fdfea88d7 100644 --- a/rust/exo_net/tests/test_python.py +++ b/rust/exo_net/tests/test_python.py @@ -45,5 +45,6 @@ async def _await_recv(h: NetworkingHandle): def scoped_lock_file(): a = Pidfile("/tmp/lock.pid", 0o0600) + if __name__ == "__main__": asyncio.run(test_sleep_on_multiple_items()) diff --git a/rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi b/rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi deleted file mode 100644 index ed5d0e52c..000000000 --- a/rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi +++ /dev/null @@ -1,11 +0,0 @@ -# This file is automatically generated by pyo3_stub_gen -# ruff: noqa: E501, F401 - -import typing - -@typing.final -class StateProxy: - @staticmethod - async def init() -> StateProxy: ... - async def snapshot(self) -> str: ... - diff --git a/rust/networking/Cargo.toml b/rust/networking/Cargo.toml index bcb7d9fc7..fa8011ac7 100644 --- a/rust/networking/Cargo.toml +++ b/rust/networking/Cargo.toml @@ -4,20 +4,17 @@ version.workspace = true edition.workspace = true [dependencies] -async-stream = "0.3.6" +async-stream.workspace = true futures-lite.workspace = true -netwatcher = { version = "0.6.0", features = ["tokio"] } -parking_lot = "0.12.5" +netwatcher = { workspace = true, features = ["tokio"] } +parking_lot.workspace = true tokio = { workspace = true, features = ["full"] } -zenoh = { version = "=1.9.0", features = ["internal", "plugins", "unstable"] } -zenoh-plugin-storage-manager = { version = "=1.9.0", default-features = false } -zenoh-plugin-trait = "=1.9.0" -zerompk = { version = "0.4.2", features = ["derive"] } -rand = "0.10.1" -tracing = "0.1.44" +zenoh = { workspace = true, features = ["internal", "plugins", "unstable"] } +zenoh-plugin-storage-manager.workspace = true +zenoh-plugin-trait.workspace = true +rand.workspace = true log.workspace = true bytemuck = { workspace = true, features = ["derive"] } [lints] workspace = true - diff --git a/rust/networking/examples/put_string.rs b/rust/networking/examples/put_string.rs index aa1888d5c..a21ad4748 100644 --- a/rust/networking/examples/put_string.rs +++ b/rust/networking/examples/put_string.rs @@ -1,5 +1,5 @@ +use log::info; use networking; -use tracing::info; use zenoh::Result; #[tokio::main] @@ -9,14 +9,15 @@ async fn main() -> Result<()> { let cfg = networking::cfg(rand::random(), 0)?; let session = networking::open(cfg, 52414).await?; let _tok = session + .z .liveliness() - .declare_token(format!("nodes/{}/live", session.zid())) + .declare_token(format!("nodes/{}/live", session.z.zid())) .await?; let key_expr = "storage/mem1/name"; let payload = "me"; info!("Putting Data ('{key_expr}': '{payload}')..."); - session.put(key_expr, payload).await?; + session.z.put(key_expr, payload).await?; tokio::signal::ctrl_c().await?; Ok(()) } diff --git a/rust/networking/examples/serve_storage.rs b/rust/networking/examples/serve_storage.rs index adb815a35..4267bf8b7 100644 --- a/rust/networking/examples/serve_storage.rs +++ b/rust/networking/examples/serve_storage.rs @@ -1,5 +1,5 @@ +use log::info; use networking; -use tracing::info; use zenoh::Result; #[tokio::main] @@ -9,10 +9,12 @@ async fn main() -> Result<()> { let cfg = networking::cfg(rand::random(), 52414)?; let session = networking::open(cfg, 52414).await?; let _tok = session + .z .liveliness() - .declare_token(format!("nodes/{}/live", session.zid())) + .declare_token(format!("nodes/{}/live", session.z.zid())) .await?; let _sub = session + .z .liveliness() .declare_subscriber("nodes/*/live") .history(true) diff --git a/rust/networking/src/lib.rs b/rust/networking/src/lib.rs index df965d6c5..7a35dc312 100644 --- a/rust/networking/src/lib.rs +++ b/rust/networking/src/lib.rs @@ -49,7 +49,7 @@ pub async fn open(cfg: zenoh::Config, listen_port: u16) -> Result { .plugins_manager(plugins) .build() .await?; - let session = zenoh::session::init(runtime.clone().into()).await?; + let z = zenoh::session::init(runtime.clone().into()).await?; runtime.start().await?; let mut discovery = Discovery::new(z.zid(), listen_port).await?; let _jh = tokio::task::spawn(async move { @@ -82,7 +82,7 @@ pub async fn open(cfg: zenoh::Config, listen_port: u16) -> Result { } pub struct Session { - pub session: ZSession, + pub z: ZSession, _jh: JoinHandle<()>, } impl Drop for Session { @@ -90,4 +90,3 @@ impl Drop for Session { self._jh.abort(); } } - diff --git a/rust/networking/src/swarm.rs b/rust/networking/src/swarm.rs index a49caaeaf..4d9ae9e46 100644 --- a/rust/networking/src/swarm.rs +++ b/rust/networking/src/swarm.rs @@ -6,7 +6,6 @@ use std::pin::Pin; use futures_lite::Stream; use tokio::sync::mpsc; use tokio::sync::oneshot; -use tracing::info; use zenoh::Result; use zenoh::Session; use zenoh::handlers::FifoChannelHandler; @@ -14,7 +13,6 @@ use zenoh::liveliness::LivelinessToken; use zenoh::pubsub::Subscriber; use zenoh::sample::Sample; use zenoh::sample::SampleKind; -use zerompk::{FromMessagePack, ToMessagePack}; #[derive(Debug)] pub enum ToSwarm { @@ -32,7 +30,7 @@ pub enum ToSwarm { result_sender: oneshot::Sender>, }, } -#[derive(Debug, ToMessagePack, FromMessagePack)] +#[derive(Debug)] pub enum FromSwarm { Message { topic: String, data: Vec }, Discovered {}, @@ -41,26 +39,26 @@ pub enum FromSwarm { pub type Topics = HashMap>; pub struct Swarm { - cfg: zenoh::Config, + pub session: crate::Session, from_client: mpsc::Receiver, } impl Swarm { pub fn into_stream(self) -> Pin + Send>> { let Swarm { - cfg, + session, mut from_client, } = self; let stream = async_stream::stream! { + let mut session = session; let (mut to_topics, mut from_topics) = mpsc::channel(1024); let mut topics = Topics::new(); - let Ok(mut session) = crate::open(cfg).await else { return; }; - let Ok((_token, discovery)) = register_liveness(&mut session).await else { return; }; + let Ok((_token, discovery)) = register_liveness(&mut session.z).await else { return; }; loop { tokio::select! { msg = from_client.recv() => { let Some(msg) = msg else { break }; - on_message(&mut session, &mut topics, &mut to_topics, msg).await; + on_message(&mut session.z, &mut topics, &mut to_topics, msg).await; } event = from_topics.recv() => { if let Some(event) = event { @@ -73,11 +71,11 @@ impl Swarm { let nid = key_expr.strip_prefix("nodes/").and_then(|s| s.strip_suffix("/live")); yield match token.kind() { SampleKind::Put => { - info!("discovered: {nid:?}"); + log::info!("discovered: {nid:?}"); FromSwarm::Discovered {} } SampleKind::Delete => { - info!("expired: {nid:?}"); + log::info!("expired: {nid:?}"); FromSwarm::Expired {} } } @@ -171,7 +169,7 @@ async fn on_message( } } -pub fn create_swarm( +pub async fn create_swarm( identity: u128, from_client: mpsc::Receiver, bootstrap_peers: Vec, diff --git a/rust/parts.nix b/rust/parts.nix index f5272b72e..ae7507485 100644 --- a/rust/parts.nix +++ b/rust/parts.nix @@ -55,6 +55,7 @@ ]; OPENSSL_NO_VENDOR = "1"; + MATURIN_NO_INSTALL_RUST = "1"; # Required for pyo3 tests to find libpython LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.python313 ]; @@ -81,11 +82,11 @@ config = { packages = { # Python bindings wheel via maturin - exo_pyo3_bindings = craneLib.buildPackage ( + exo-net = craneLib.buildPackage ( commonArgs // { inherit cargoArtifacts; - pname = "exo_pyo3_bindings"; + pname = "exo-net"; nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ pkgs.maturin @@ -95,7 +96,7 @@ maturin build \ --release \ --manylinux off \ - --manifest-path rust/exo_pyo3_bindings/Cargo.toml \ + --manifest-path rust/exo_net/Cargo.toml \ --features "pyo3/extension-module,pyo3/experimental-async" \ --interpreter ${pkgs.python313}/bin/python \ --out dist diff --git a/src/exo/api/main.py b/src/exo/api/main.py index 9e23c3194..fcd54c931 100644 --- a/src/exo/api/main.py +++ b/src/exo/api/main.py @@ -1913,7 +1913,6 @@ class API: try: async with self._tg as tg: logger.info("Starting API") - tg.start_soon(self.transport.run) tg.start_soon(self._apply_state) tg.start_soon(self._pause_on_new_election) tg.start_soon(self._cleanup_expired_images) diff --git a/src/exo/main.py b/src/exo/main.py index 86a46561b..798d77aa3 100644 --- a/src/exo/main.py +++ b/src/exo/main.py @@ -50,8 +50,8 @@ class Node: @classmethod async def create(cls, args: "Args") -> Self: - keypair = get_node_id_keypair() - node_id = NodeId(keypair.to_node_id()) + keypair = os.urandom(16) + node_id = NodeId(keypair.hex()) session_id = SessionId(master_node_id=node_id, election_clock=0) router = Router.create( keypair, diff --git a/src/exo/routing/connection_message.py b/src/exo/routing/connection_message.py index ddcd143ba..0d7b69fc8 100644 --- a/src/exo/routing/connection_message.py +++ b/src/exo/routing/connection_message.py @@ -1,4 +1,4 @@ -from exo_pyo3_bindings import PyFromSwarm +from exo_net import PyFromSwarm from exo.utils.pydantic_ext import FrozenModel diff --git a/src/exo/routing/router.py b/src/exo/routing/router.py index 4ef2311c3..31a7710b9 100644 --- a/src/exo/routing/router.py +++ b/src/exo/routing/router.py @@ -2,8 +2,6 @@ from collections.abc import Sequence from copy import copy from itertools import count from math import inf -from os import PathLike -from pathlib import Path from typing import cast from anyio import ( @@ -12,15 +10,12 @@ from anyio import ( move_on_after, sleep_forever, ) -from exo_pyo3_bindings import ( - Keypair, +from exo_net import ( NetworkingHandle, PyFromSwarm, ) -from filelock import FileLock from loguru import logger -from exo.shared.constants import EXO_NODE_ID_KEYPAIR from exo.utils.channels import Receiver, Sender, channel from exo.utils.pydantic_ext import FrozenModel from exo.utils.task_group import TaskGroup @@ -102,12 +97,12 @@ class Router: @classmethod def create( cls, - identity: Keypair, + identity: bytes, bootstrap_peers: Sequence[str] = (), listen_port: int = 0, ) -> "Router": return cls( - handle=NetworkingHandle(identity, list(bootstrap_peers), listen_port) + handle=NetworkingHandle.new(identity, list(bootstrap_peers), listen_port) ) def __init__(self, handle: NetworkingHandle): @@ -189,9 +184,7 @@ class Router: logger.debug(from_swarm) match from_swarm: case PyFromSwarm.Message(topic, data): - logger.trace( - f"Received message on {topic} with payload {data}" - ) + logger.trace(f"Received message on {topic} with payload {data}") if topic not in self.topic_routers: logger.warning( f"Received message on unknown or inactive topic {topic}" diff --git a/tmp/old_tests/headless_runner.py b/tmp/old_tests/headless_runner.py deleted file mode 100644 index 176f6fcf8..000000000 --- a/tmp/old_tests/headless_runner.py +++ /dev/null @@ -1,264 +0,0 @@ -import socket -from typing import Literal - -import anyio -from fastapi import FastAPI -from fastapi.responses import Response, StreamingResponse -from hypercorn import Config -from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType] -from loguru import logger -from pydantic import BaseModel - -from exo.shared.constants import EXO_DEFAULT_MODELS_DIR -from exo.shared.models.model_cards import ModelCard, ModelId -from exo.shared.types.chunks import TokenChunk -from exo.shared.types.commands import CommandId -from exo.shared.types.common import Host, NodeId -from exo.shared.types.events import ChunkGenerated, Event, RunnerStatusUpdated -from exo.shared.types.tasks import ( - ConnectToGroup, - LoadModel, - Shutdown, - StartWarmup, - Task, - TextGeneration, -) -from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams -from exo.shared.types.worker.instances import ( - BoundInstance, - Instance, - InstanceId, - MlxJacclInstance, - MlxRingInstance, -) -from exo.shared.types.worker.runners import ( - RunnerFailed, - RunnerId, - RunnerShutdown, - ShardAssignments, -) -from exo.shared.types.worker.shards import PipelineShardMetadata, TensorShardMetadata -from exo.utils.channels import channel, mp_channel -from exo.utils.info_gatherer.info_gatherer import GatheredInfo, InfoGatherer -from exo.worker.runner.bootstrap import entrypoint - - -class Tests(BaseModel): - # list[hostname, ip addr] - devs: list[list[str]] - ibv_devs: list[list[str | None]] | None - model_id: ModelId - kind: Literal["ring", "jaccl", "both"] - - -iid = InstanceId("im testing here") - - -async def main(): - logger.info("starting cool server majig") - cfg = Config() - cfg.bind = "0.0.0.0:52414" - # nb: shared.logging needs updating if any of this changes - cfg.accesslog = "-" - cfg.errorlog = "-" - ev = anyio.Event() - app = FastAPI() - app.post("/run_test")(run_test) - app.post("/kill")(lambda: kill(ev)) - app.get("/tb_detection")(tb_detection) - app.get("/models")(list_models) - await serve( - app, # type: ignore - cfg, - shutdown_trigger=lambda: ev.wait(), - ) - - -def kill(ev: anyio.Event): - ev.set() - return Response(status_code=204) - - -async def tb_detection(): - send, recv = channel[GatheredInfo]() - ig = InfoGatherer(send) - with anyio.move_on_after(1): - await ig._monitor_system_profiler_thunderbolt_data() # pyright: ignore[reportPrivateUsage] - with recv: - return recv.collect() - - -def list_models(): - sent = set[str]() - for path in EXO_DEFAULT_MODELS_DIR.rglob("model-*.safetensors"): - if "--" not in path.parent.name: - continue - name = path.parent.name.replace("--", "/") - if name in sent: - continue - sent.add(name) - yield ModelId(path.parent.name.replace("--", "/")) - - -async def run_test(test: Tests): - weird_hn = socket.gethostname() - for dev in test.devs: - if weird_hn.startswith(dev[0]) or dev[0].startswith(weird_hn): - hn = dev[0] - break - else: - raise ValueError(f"{weird_hn} not in {test.devs}") - - async def run(): - logger.info(f"testing {test.model_id}") - - instances: list[Instance] = [] - if test.kind in ["ring", "both"]: - i = await ring_instance(test, hn) - if i is None: - yield "no model found" - return - instances.append(i) - if test.kind in ["jaccl", "both"]: - i = await jaccl_instance(test) - if i is None: - yield "no model found" - return - instances.append(i) - - for instance in instances: - recv = await execute_test(test, instance, hn) - - str_out = "" - - for item in recv: - if isinstance(item, ChunkGenerated): - assert isinstance(item.chunk, TokenChunk) - str_out += item.chunk.text - - if isinstance(item, RunnerStatusUpdated) and isinstance( - item.runner_status, (RunnerFailed, RunnerShutdown) - ): - yield str_out + "\n" - yield item.model_dump_json() + "\n" - - return StreamingResponse(run()) - - -async def ring_instance(test: Tests, hn: str) -> Instance | None: - hbn = [Host(ip="198.51.100.0", port=52417) for _ in test.devs] - world_size = len(test.devs) - for i in range(world_size): - if test.devs[i][0] == hn: - hn = test.devs[i][0] - hbn[(i - 1) % world_size] = Host(ip=test.devs[i - 1][1], port=52417) - hbn[(i + 1) % world_size] = Host(ip=test.devs[i + 1][1], port=52417) - hbn[i] = Host(ip="0.0.0.0", port=52417) - break - else: - raise ValueError(f"{hn} not in {test.devs}") - - card = await ModelCard.load(test.model_id) - instance = MlxRingInstance( - instance_id=iid, - ephemeral_port=52417, - hosts_by_node={NodeId(hn): hbn}, - shard_assignments=ShardAssignments( - model_id=test.model_id, - node_to_runner={NodeId(host[0]): RunnerId(host[0]) for host in test.devs}, - runner_to_shard={ - RunnerId(test.devs[i][0]): PipelineShardMetadata( - model_card=card, - device_rank=i, - world_size=world_size, - start_layer=(card.n_layers // world_size) * i, - end_layer=min( - card.n_layers, (card.n_layers // world_size) * (i + 1) - ), - n_layers=min(card.n_layers, (card.n_layers // world_size) * (i + 1)) - - (card.n_layers // world_size) * i, - ) - for i in range(world_size) - }, - ), - ) - - return instance - - -async def execute_test(test: Tests, instance: Instance, hn: str) -> list[Event]: - world_size = len(test.devs) - commands: list[Task] = [ - (LoadModel(instance_id=iid)), - (StartWarmup(instance_id=iid)), - ( - TextGeneration( - task_params=TextGenerationTaskParams( - model=test.model_id, - instructions="You are a helpful assistant", - input=[ - InputMessage( - role="user", content="What is the capital of France?" - ) - ], - ), - command_id=CommandId("yo"), - instance_id=iid, - ) - ), - (Shutdown(runner_id=RunnerId(hn), instance_id=iid)), - ] - if world_size > 1: - commands.insert(0, ConnectToGroup(instance_id=iid)) - bound_instance = BoundInstance( - instance=instance, bound_runner_id=RunnerId(hn), bound_node_id=NodeId(hn) - ) - ev_send, _ev_recv = mp_channel[Event]() - task_send, task_recv = mp_channel[Task]() - - for command in commands: - task_send.send(command) - - entrypoint( - bound_instance, - ev_send, - task_recv, - logger, - ) - - # TODO(evan): return ev_recv.collect() - return [] - - -async def jaccl_instance(test: Tests) -> MlxJacclInstance | None: - card = await ModelCard.load(test.model_id) - world_size = len(test.devs) - assert test.ibv_devs - - return MlxJacclInstance( - instance_id=iid, - jaccl_devices=test.ibv_devs, - # rank 0 is always coordinator - jaccl_coordinators={ - NodeId(host[0]): test.devs[0][1] + ":52417" for host in test.devs - }, - shard_assignments=ShardAssignments( - model_id=test.model_id, - node_to_runner={NodeId(host[0]): RunnerId(host[0]) for host in test.devs}, - runner_to_shard={ - RunnerId(host[0]): TensorShardMetadata( - model_card=card, - device_rank=i, - world_size=world_size, - start_layer=0, - end_layer=card.n_layers, - n_layers=card.n_layers, - ) - for i, host in enumerate(test.devs) - }, - ), - ) - - -if __name__ == "__main__": - anyio.run(main) diff --git a/tmp/old_tests/start_distributed_test.py b/tmp/old_tests/start_distributed_test.py deleted file mode 100755 index bf11c73c6..000000000 --- a/tmp/old_tests/start_distributed_test.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -import itertools -import json -import subprocess -import sys -from concurrent.futures import ThreadPoolExecutor -from typing import Any, cast -from urllib.request import Request, urlopen - -if not (args := sys.argv[1:]): - sys.exit( - f"USAGE: {sys.argv[0]} [host1] [host2] ...\nkind is optional, and should be jaccl or ring" - ) - -kind = args[0] if args[0] in ("jaccl", "ring") else "both" -hosts = args[1:] if kind != "both" else args -ts = subprocess.run( - ["tailscale", "status"], check=True, text=True, capture_output=True -).stdout.splitlines() -ip = {sl[1]: sl[0] for line in ts if len(sl := line.split()) >= 2} -ips = [ip[h] for h in hosts] -devs = [[h, ip[h]] for h in hosts] -n = len(hosts) - - -def get_tb(a: str) -> list[dict[str, Any]]: - with urlopen(f"http://{a}:52414/tb_detection", timeout=5) as r: # pyright: ignore[reportAny] - return json.loads(r.read()) # pyright: ignore[reportAny] - - -def get_models(a: str) -> set[str]: - with urlopen(f"http://{a}:52414/models", timeout=5) as r: # pyright: ignore[reportAny] - return set(json.loads(r.read())) # pyright: ignore[reportAny] - - -def run(h: str, a: str, body: bytes) -> None: - with urlopen( - Request( - f"http://{a}:52414/run_test", - data=body, - method="POST", - headers={"Content-Type": "application/json"}, - ), - timeout=300, - ) as r: # pyright: ignore[reportAny] - for line in r.read().decode(errors="replace").splitlines(): # pyright: ignore[reportAny] - print(f"\n{h}@{a}: {line}", flush=True) - - -with ThreadPoolExecutor(n) as exctr: - if kind in ("jaccl", "both"): - payloads = list(exctr.map(get_tb, ips)) - - u2e = { - ident["domainUuid"]: (i, ident["rdmaInterface"]) - for i, p in enumerate(payloads) - for d in p - for ident in cast( - list[dict[str, str]], - d.get("MacThunderboltIdentifiers", {}).get("idents", []), # pyright: ignore[reportAny] - ) - } - edges = { - (u2e[s][0], u2e[t][0]): u2e[t][1] - for p in payloads - for d in p - for c in d.get("MacThunderboltConnections", {}).get("conns", []) # pyright: ignore[reportAny] - if (s := c["sourceUuid"]) in u2e and (t := c["sinkUuid"]) in u2e # pyright: ignore[reportAny] - } - ibv_devs = [[edges.get((i, j)) for j in range(n)] for i in range(n)] - else: - ibv_devs = None - - models = set[str].intersection(*exctr.map(get_models, ips)) - - print("\n") - print("=" * 70) - print(f"Starting test with {models}") - print("=" * 70) - print("\n") - for model in models: - body = json.dumps( - {"devs": devs, "model_id": model, "ibv_devs": ibv_devs, "kind": kind} - ).encode() - list(exctr.map(run, hosts, ips, itertools.repeat(body))) diff --git a/tmp/old_tests/run_exo_on.sh b/tmp/run_exo_on.sh similarity index 100% rename from tmp/old_tests/run_exo_on.sh rename to tmp/run_exo_on.sh diff --git a/tmp/state_proxy.py b/tmp/state_proxy.py deleted file mode 100644 index 8195d83e8..000000000 --- a/tmp/state_proxy.py +++ /dev/null @@ -1,13 +0,0 @@ -from exo_pyo3_bindings import StateProxy -import anyio - -async def main(): - sp = await StateProxy.init() - while True: - data = await sp.snapshot() - if data != "{}": - print(data) - await anyio.sleep(1) - -if __name__ == "__main__": - anyio.run(main) diff --git a/uv.lock b/uv.lock index 4f568a523..171db44ea 100644 --- a/uv.lock +++ b/uv.lock @@ -24,7 +24,7 @@ prerelease-mode = "allow" members = [ "exo", "exo-bench", - "exo-pyo3-bindings", + "exo-net", "exo-tools", ] overrides = [{ name = "opencv-python", marker = "python_full_version < '0'" }] @@ -622,9 +622,9 @@ requires-dist = [ ] [[package]] -name = "exo-pyo3-bindings" -version = "0.2.10" -source = { editable = "rust/exo_pyo3_bindings" } +name = "exo-net" +version = "0.3.0" +source = { editable = "rust/exo_net" } [package.dev-dependencies] dev = [ @@ -637,7 +637,7 @@ dev = [ [package.metadata.requires-dev] dev = [ - { name = "exo-pyo3-bindings", editable = "rust/exo_pyo3_bindings" }, + { name = "exo-net", editable = "rust/exo_net" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.0.0" }, ]