it builds

.envrc

@@ -1 +1,8 @@
 use flake
+
+# creates .venv if doesn't exist and loads its environment
+export VIRTUAL_ENV=".venv"
+if ! [ -d "./$VIRTUAL_ENV" ]; then
+  uv venv
+fi
+layout python

.gitignore

@@ -38,8 +38,6 @@ bench/**/*.json
 # tmp
 tmp/models
 /build/exo
-/.agents
 /.claude/skills
 /.claude
 /.codex
-skills-lock.json

AGENTS.md

@@ -4,7 +4,7 @@ This file provides guidance to AI coding agents when working with code in this r
 
 ## Project Overview
 
-exo is a distributed AI inference system that connects multiple devices into a cluster. It enables running large language models across multiple machines using MLX as the inference backend and zenoh for peer-to-peer networking.
+exo is a distributed AI inference system that connects multiple devices into a cluster. It enables running large language models across multiple machines using MLX as the inference backend and libp2p for peer-to-peer networking.
 
 ## Build & Run Commands
 
@@ -69,7 +69,7 @@ If `nix fmt` changes any files, stage them before committing. The CI runs `nix f
 
 ### Node Composition
 A single exo `Node` (src/exo/main.py) runs multiple components:
-- **Router**: zenoh-based pub/sub messaging via Rust bindings (exo_rs)
+- **Router**: libp2p-based pub/sub messaging via Rust bindings (exo_pyo3_bindings)
 - **Worker**: Handles inference tasks, downloads models, manages runner processes
 - **Master**: Coordinates cluster state, places model instances across nodes
 - **Election**: Bully algorithm for master election
@@ -81,7 +81,7 @@ Components communicate via typed pub/sub topics (src/exo/routing/topics.py):
 - `LOCAL_EVENTS`: Workers send events to master for indexing
 - `COMMANDS`: Workers/API send commands to master
 - `ELECTION_MESSAGES`: Election protocol messages
-- `CONNECTION_MESSAGES`: zenoh connection updates
+- `CONNECTION_MESSAGES`: libp2p connection updates
 
 ### Event Sourcing
 The system uses event sourcing for state management:
@@ -98,8 +98,8 @@ The system uses event sourcing for state management:
 
 ### Rust Components
 Rust code in `rust/` provides:
-- `networking`: zenoh networking (gossipsub, peer discovery)
-- `exo_rs`: PyO3 bindings exposing Rust to Python
+- `networking`: libp2p networking (gossipsub, peer discovery)
+- `exo_pyo3_bindings`: PyO3 bindings exposing Rust to Python
 - `system_custodian`: System-level operations
 
 ### Dashboard

CONTRIBUTING.md

@@ -29,7 +29,6 @@ To run EXO from source:
 git clone https://github.com/exo-explore/exo.git
 cd exo/dashboard
 npm install && npm run build && cd ..
-uv sync --extra mlx
 uv run exo
 ```
 

Cargo.toml

@@ -1,6 +1,11 @@
 [workspace]
 resolver = "3"
-members = ["rust/exo_rs", "rust/networking"]
+members = [
+    "rust/networking",
+    "rust/exo_pyo3_bindings",
+    "rust/util",
+    "rust/babblerd",
+]
 
 [workspace.package]
 version = "0.0.1"
@@ -20,73 +25,30 @@ opt-level = 3
 [workspace.dependencies]
 ## Crate members as common dependencies
 networking = { path = "rust/networking" }
+util = { path = "rust/util" }
 
-# pyo3
-pyo3 = "0.28.3"
-pyo3-async-runtimes = "0.28.0"
-pyo3-log = "0.13.3"
-pyo3-stub-gen = "0.22.3"
-
-# util
+# Macro dependecies
 extend = "1.2"
 delegate = "0.13"
 
 # Utility dependencies
-nix = "0.31"
+keccak-const = "0.2"
 
 # Async dependencies
+async-stream = "0.3"
 tokio = "1.46"
 futures-lite = "2.6.1"
-async-stream = "0.3.6"
-pin-project = "1.1.10"
-serde_json = "1.0.149"
-rand = "0.10.1"
-parking_lot = "0.12.5"
+futures-timer = "3.0"
+
+# Data structures
+either = "1.15"
 
 # Tracing/logging
 log = "0.4"
-env_logger = "0.11.10"
 
 # networking
-zenoh = "=1.9.0"
-zenoh-plugin-storage-manager = { version = "=1.9.0", default-features = false }
-zenoh-plugin-trait = "=1.9.0"
-netwatcher = "0.6.0"
-bytemuck = "1.25.0"
-blake3 = "1.8.5"
-smol = "2.0.2"
-socket2 = "0.6.4"
-tracing = "0.1.44"
-pidfile-rs = { git = "https://github.com/AndreiCravtov/pidfile-rs" }
-
-[patch.crates-io]
-zenoh = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-buffers = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-codec = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-collections = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-config = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-core = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-crypto = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-keyexpr = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-commons = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-quic = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-quic_datagram = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-tcp = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-tls = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-udp = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-unixsock_stream = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-link-ws = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-macros = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-plugin-trait = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-protocol = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-result = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-runtime = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-sync = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-task = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-transport = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh-util = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
-zenoh_backend_traits = { git = "https://github.com/evanev7/zenoh.git", branch = "exo" }
+libp2p = "0.56"
+libp2p-tcp = "0.44"
 
 [workspace.lints.rust]
 static_mut_refs = "warn"      # Or use "warn" instead of deny

MISSED_THINGS.md

@@ -0,0 +1,41 @@
+# Missed things
+[X] Log EXO_LIBP2P_NAMESPACE on start in exo/main.py
+[X] Ordering of warmup was changed, which is wrong. It was changed to rank < n-1, then rank=n-1. It should be rank!=0 then rank=0 (this matches the auto_parallel implementation. NOTE: we use a different convention to mlx-lm, our terminal rank is rank=n-1 whereas mlx-lm is rank=0 hence i can see why this was changed wrongly).
+[X] Downloads keying by model_id not shard_metadata (worker/plan.py, worker/main.py).
+[X] Fetching download status of all models on start
+[X] Deduplication of tasks in plan_step.
+[X] resolve_allow_patterns should just be wildcard now.
+[X] no mx_barrier in genreate.py mlx_generate at the end.
+[] cache assertion not needed in auto_parallel.py PipelineLastLayer.
+[X] GPTOSS support dropped in auto_parallel.py.
+[X] sharding changed "all-to-sharded" became _all_to_sharded in auto_parallel.py.
+[X] same as above with "sharded-to-all" became _sharded_to_all in auto_parallel.py.
+[X] Dropped support for Ministral3Model, DeepseekV32Model, Glm4MoeModel, Qwen3NextModel, GptOssMode in auto_parallel.py.
+[] Dropped prefill/decode code in auto_parallel.py and utils_mlx.py.
+[X] KV_CACHE_BITS should be None to disable quantized KV cache.
+[X] Dropped _set_nofile_limit in utils_mlx.py.
+[X] We have group optional in load_mlx_items in utils_mlx.py.
+[X] Dropped add_missing_chat_templates for GptOss in load_mlx_items in utils_mlx.py.
+[X] Dropped model.make_cache in make_kv_cache in utils_mlx.py.
+[X] We put cache limit back in utils_mlx.py.
+[X] topology.py remove_node removes the connections after checking if node is is in self._node_id_to_rx_id_map. on beta_1 it checks after, so would remove stale connections I guess?
+[X] Missing Glm 4.7 model cards (this isn't ready yet but should be picked up, probably create an issue... the blocker is transforemrs version doesn't support the tokenizer for Glm 4.7. rc-1 does but we can't upgrade as it breaks other things.)
+[] try-except in _command_processor only excepts ValueError. This was silently failing leading to un-debuggable errors (we had a KeyError that was happening ). Changed this to catch Exception instead of ValueError. See exo-v2 89ae38405e0052e3c22405daf094b065878aa873 and fb99fea69b5a39017efc90c5dad0072e677455f0.
+[X] In placement.py, place_instance no longer looks at model_meta.supports_tensor and check if this tensor parallel number of nodes is supported by the model's tensor dimensions.
+[X] In placement.py, place_instanec, we no longer have the special case to exclude DeepSeek v3.1 pipeline parallel (it doesn't work).
+[] logger.warning("You have likely selected ibv for a single node instance; falling back to MlxRing") was changed to debug. That will spam this warning since it happens every time we query instance previews.
+[X] In placement_utils.py, get_mlx_jaccl_coordinators, We no longer prioritise Jaccl Coordinator IP. Now it picks the first one, which is unstable (Jaccl coordinator over TB5 is unstable).
+
+
+
+[X] Downloads keying by model_id not shard_metadata (worker/plan.py, worker/main.py).
+[X] Fetching download status of all models on start
+[X] Deduplication of tasks in plan_step.
+[X] resolve_allow_patterns should just be wildcard now.
+[X] KV_CACHE_BITS should be None to disable quantized KV cache.
+[X] We put cache limit back in utils_mlx.py.
+[X] In placement.py, place_instance no longer looks at model_meta.supports_tensor and check if this tensor parallel number of nodes is supported by the model's tensor dimensions.
+[X] In placement.py, place_instanec, we no longer have the special case to exclude DeepSeek v3.1 pipeline parallel (it doesn't work).
+[X] In placement_utils.py, get_mlx_jaccl_coordinators, We no longer prioritise Jaccl Coordinator IP. Now it picks the first one, which is unstable (Jaccl coordinator over TB5 is unstable).
+
+

README.md

@@ -118,7 +118,7 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
     --force
   ```
 
-Clone the repo, build the dashboard, install the dependencies, and run exo:
+Clone the repo, build the dashboard, and run exo:
 
 ```bash
 # Clone exo
@@ -127,9 +127,6 @@ git clone https://github.com/exo-explore/exo
 # Build dashboard
 cd exo/dashboard && npm install && npm run build && cd ..
 
-# Install Python dependencies, including the MLX backend
-uv sync --extra mlx
-
 # Run exo
 uv run exo
 ```
@@ -179,7 +176,7 @@ rustup toolchain install nightly
 
 **Note:** The `macmon` package is macOS-only and not required for Linux.
 
-Clone the repo, build the dashboard, install the dependencies, and run exo:
+Clone the repo, build the dashboard, and run exo:
 
 ```bash
 # Clone exo
@@ -188,10 +185,6 @@ git clone https://github.com/exo-explore/exo
 # Build dashboard
 cd exo/dashboard && npm install && npm run build && cd ..
 
-# Install Python dependencies with the MLX backend for your hardware
-# (NVIDIA: --extra mlx-cuda13 or --extra mlx-cuda12)
-uv sync --extra mlx-cpu
-
 # Run exo
 uv run exo
 ```
@@ -208,12 +201,6 @@ This starts the exo dashboard and API at http://localhost:52415/
   uv run exo --no-worker
   ```
 
-- `--legacy-daemon`: Run exo as a legacy SysV-style background daemon using double-fork daemonization. This is intended for legacy init scripts; systemd and launchd should run exo in the foreground without this flag.
-
-  ```bash
-  uv run exo --legacy-daemon
-  ```
-
 **File Locations (Linux):**
 
 exo follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) on Linux:
@@ -236,12 +223,6 @@ The macOS app requires macOS Tahoe 26.2 or later.
 
 Download the latest build here: [EXO-latest.dmg](https://assets.exolabs.net/EXO-latest.dmg).
 
-You can also install the latest build with Homebrew:
-
-```bash
-brew install --cask exo
-```
-
 The app will ask for permission to modify system settings and install a new Network profile. Improvements to this are being worked on.
 
 **Custom Namespace for Cluster Isolation:**
@@ -414,18 +395,6 @@ Sample response:
 }
 ```
 
-This command is asynchronous. Before sending inference requests, wait until the
-API sees the new instance for this model:
-
-```bash
-curl -N "http://localhost:52415/instance/await?model_id=mlx-community/Llama-3.2-1B-Instruct-4bit"
-```
-
-The endpoint returns an SSE stream. A successful wait emits a message with
-`"type": "ready"` and the matching instance; a timeout emits `"type": "timeout"`.
-By default it waits indefinitely. Set `timeout_seconds` to a positive value to
-bound the wait.
-
 ---
 
 **3. Send a chat completion**

TODO.md

@@ -1,13 +1,14 @@
-1. EXO_BOOTSTRAP_PEERS is currently broken
-
+3. Task cancellation. When API http request gets cancelled, it should cancel corresponding task.
 4. I'd like to see profiled network latency / bandwidth.
 5. I'd like to see how much bandwidth each link is using.
 7. Solve the problem of in continuous batching when a new prompt comes in, it will block decode of the current batch until the prefill is complete.
 8. We want people to be able to copy models over to a new device without ever connecting EXO to the internet. Right now EXO require internet connection once to cache some files to check if a download is complete. Instead, we should simply check if there is a non-empty model folder locally with no .partial files. This indicates it's a fully downloaded model that can be loaded.
 13. Memory pressure instead of memory used.
 14. Show the type of each connection (TB5, Ethernet, etc.) in the UI. Refer to old exo: https://github.com/exo-explore/exo/blob/56f783b38dc6b08ce606b07a5386dc40dae00330/exo/helpers.py#L251
+15. Prioritise certain connection types (or by latency). TB5 > Ethernet > WiFi. Refer to old exo: https://github.com/exo-explore/exo/blob/56f783b38dc6b08ce606b07a5386dc40dae00330/exo/helpers.py#L251
 16. Dynamically switch to higher priority connection when it becomes available. Probably bring back InstanceReplacedAtomically.
 17. Faster model loads by streaming model from other devices in cluster.
 18. Add support for specifying the type of network connection to use in a test. Depends on 15/16.
+25. Rethink retry logic
 27. Log cleanup - per-module log filters and default to DEBUG log levels
 28. Validate RDMA connections with ibv_devinfo in the info gatherer

app/EXO/EXO/ExoProcessController.swift

@@ -352,7 +352,7 @@ final class ExoProcessController: ObservableObject {
     private func makeEnvironment(for runtimeURL: URL) -> [String: String] {
         var environment = ProcessInfo.processInfo.environment
         environment["EXO_RUNTIME_DIR"] = runtimeURL.path
-        environment["EXO_ZENOH_NAMESPACE"] = computeNamespace()
+        environment["EXO_LIBP2P_NAMESPACE"] = computeNamespace()
         if !hfToken.isEmpty {
             environment["HF_TOKEN"] = hfToken
         }

app/EXO/EXO/Services/ClusterStateService.swift

@@ -37,7 +37,7 @@ final class ClusterStateService: ObservableObject {
     /// gain nothing from being cached on disk. Use an ephemeral session
     /// with `urlCache = nil` so neither response bodies nor metadata
     /// touch disk.
-    nonisolated private static func makeNonCachingSession() -> URLSession {
+    private static func makeNonCachingSession() -> URLSession {
         let config = URLSessionConfiguration.ephemeral
         config.urlCache = nil
         config.requestCachePolicy = .reloadIgnoringLocalCacheData

bench/METHODOLOGY.md

@@ -125,7 +125,7 @@ A background thread polls each node at 1 Hz, collecting:
 - System power draw (W)
 - CPU cluster usage (performance and efficiency cores)
 
-**Energy** is computed via trapezoidal integration of the power samples over each inference window (the wall-clock span of each benchmark request or concurrent batch). Average power is `total_joules / total_inference_seconds`. The server additionally returns a `power_usage` block in each non-stream `/bench/chat/completions` response that splits energy into prefill and generation phases, with the boundary anchored to the first non-`PrefillProgressChunk` from the runner.
+**Energy** is computed via trapezoidal integration of the power samples over each inference window (the wall-clock span of each benchmark request or concurrent batch). Average power is `total_joules / total_inference_seconds`.
 
 ---
 
@@ -136,7 +136,6 @@ Results are written as JSON with three top-level keys:
 - **`runs`**: Array of per-request result objects, each containing:
   - `elapsed_s`, `output_text_preview` (first 200 chars)
   - `stats`: `{ prompt_tps, generation_tps, prompt_tokens, generation_tokens, peak_memory_usage }`
-  - `power_usage`: server-side total + prefill/generation split, per-node breakdown (non-stream requests only)
   - Placement metadata: `model_id`, `placement_sharding`, `placement_instance_meta`, `placement_nodes`
   - Run metadata: `pp_tokens`, `tg`, `repeat_index`, `concurrency`, `concurrent_index`
   - `download_duration_s` (if model was freshly downloaded)

bench/exo_bench.py

@@ -295,7 +295,6 @@ def run_one_completion(
         elapsed = time.perf_counter() - t0
 
         stats = out.get("generation_stats")
-        power_usage = out.get("power_usage")
         choices = out.get("choices") or [{}]
         message = choices[0].get("message", {}) if choices else {}
         content = message.get("content") or ""
@@ -331,7 +330,6 @@ def run_one_completion(
 
         elapsed = time.perf_counter() - t0
         preview = "".join(text_parts)[:200]
-        power_usage = None
 
         if not stats:
             ttft = (first_token_time - t0) if first_token_time else elapsed
@@ -350,7 +348,6 @@ def run_one_completion(
         "elapsed_s": elapsed,
         "output_text_preview": preview,
         "stats": stats,
-        "power_usage": power_usage,
     }, pp_tokens
 
 
@@ -767,7 +764,6 @@ def main() -> int:
                                 out = c.post_bench_chat_completions(_payload)
                                 elapsed = time.perf_counter() - t0
                                 stats = out.get("generation_stats")
-                                power_usage = out.get("power_usage")
                                 choices = out.get("choices") or [{}]
                                 message = (
                                     choices[0].get("message", {}) if choices else {}
@@ -777,7 +773,6 @@ def main() -> int:
                                     "elapsed_s": elapsed,
                                     "output_text_preview": text[:200],
                                     "stats": stats,
-                                    "power_usage": power_usage,
                                 }, _actual_pp
 
                             inf_t0 = time.monotonic()
@@ -873,27 +868,6 @@ def main() -> int:
                             inf_seconds = sum(t1 - t0 for t0, t1 in inference_windows)
                             avg_watts = joules / inf_seconds if inf_seconds > 0 else 0
                             summary += f"    energy={joules:.1f}J ({avg_watts:.1f}W avg over {inf_seconds:.1f}s inference)"
-
-                        # mean() not sum() across concurrent runs: each
-                        # request's PowerSampler observes the same shared
-                        # cluster state, so they all report the same figure.
-                        prefill_energies = [
-                            (x.get("power_usage") or {}).get("prefill_energy_joules")
-                            for x in runs
-                        ]
-                        gen_energies = [
-                            (x.get("power_usage") or {}).get("generation_energy_joules")
-                            for x in runs
-                        ]
-                        prefill_vals = [e for e in prefill_energies if e is not None]
-                        gen_vals = [e for e in gen_energies if e is not None]
-                        if prefill_vals and gen_vals:
-                            avg_pref = mean(prefill_vals)
-                            avg_gen = mean(gen_vals)
-                            summary += (
-                                f"    prefill_energy={avg_pref:.1f}J  "
-                                f"gen_energy={avg_gen:.1f}J"
-                            )
                         logger.info(f"{summary}\n")
                     time.sleep(2)
         finally:

dashboard/package-lock.json

@@ -8,7 +8,6 @@
 			"name": "exo-dashboard",
 			"version": "1.0.0",
 			"dependencies": {
-				"devalue": "^5.6.2",
 				"highlight.js": "^11.11.1",
 				"katex": "^0.16.27",
 				"marked": "^17.0.1",
@@ -2332,9 +2331,9 @@
 			}
 		},
 		"node_modules/devalue": {
-			"version": "5.6.2",
-			"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.6.2.tgz",
-			"integrity": "sha512-nPRkjWzzDQlsejL1WVifk5rvcFi/y1onBRxjaFMjZeR9mFpqu2gmAZ9xUB9/IEanEP/vBtGeGganC/GO1fmufg==",
+			"version": "5.5.0",
+			"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.5.0.tgz",
+			"integrity": "sha512-69sM5yrHfFLJt0AZ9QqZXGCPfJ7fQjvpln3Rq5+PS03LD32Ost1Q9N+eEnaQwGRIriKkMImXD56ocjQmfjbV3w==",
 			"license": "MIT"
 		},
 		"node_modules/enhanced-resolve": {

dashboard/package.json

@@ -11,6 +11,8 @@
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json"
 	},
 	"devDependencies": {
+		"prettier": "^3.4.2",
+		"prettier-plugin-svelte": "^3.3.3",
 		"@sveltejs/adapter-static": "^3.0.10",
 		"@sveltejs/kit": "^2.48.4",
 		"@sveltejs/vite-plugin-svelte": "^5.0.0",
@@ -18,8 +20,6 @@
 		"@types/d3": "^7.4.3",
 		"@types/node": "^22",
 		"d3": "^7.9.0",
-		"prettier": "^3.4.2",
-		"prettier-plugin-svelte": "^3.3.3",
 		"svelte": "^5.0.0",
 		"svelte-check": "^4.0.0",
 		"tailwindcss": "^4.0.0",
@@ -28,7 +28,6 @@
 		"vite": "^6.0.0"
 	},
 	"dependencies": {
-		"devalue": "^5.6.2",
 		"highlight.js": "^11.11.1",
 		"katex": "^0.16.27",
 		"marked": "^17.0.1",

dashboard/src/lib/stores/app.svelte.ts

@@ -2253,9 +2253,10 @@ class AppStore {
    * @returns The model ID to use, or null if none available
    */
   private getModelForRequest(modelId?: string): string | null {
-    const requestedModelId = modelId || this.selectedChatModel;
+    if (modelId) return modelId;
+    if (this.selectedChatModel) return this.selectedChatModel;
 
-    // Only models with a placed instance can receive requests; disk downloads alone are not enough.
+    // Try to get model from first running instance
     for (const [, instanceWrapper] of Object.entries(this.instances)) {
       if (instanceWrapper && typeof instanceWrapper === "object") {
         const keys = Object.keys(instanceWrapper as Record<string, unknown>);
@@ -2263,15 +2264,8 @@ class AppStore {
           const instance = (instanceWrapper as Record<string, unknown>)[
             keys[0]
           ] as { shardAssignments?: { modelId?: string } };
-          const instanceModelId = instance?.shardAssignments?.modelId;
-
-          // ensure to only return requestedModelId that matches an instance
-          // or fall back to first instance
-          if (
-            instanceModelId &&
-            (!requestedModelId || requestedModelId === instanceModelId)
-          ) {
-            return instanceModelId;
+          if (instance?.shardAssignments?.modelId) {
+            return instance.shardAssignments.modelId;
           }
         }
       }

dashboard/src/routes/+page.svelte

@@ -1461,9 +1461,6 @@
         addToast({ type: "info", message: `Launching model...` });
         // Always auto-select the newly launched model so the user chats to what they just launched
         setSelectedChatModel(modelId);
-        userForcedIdle = false;
-        pendingChatModelId = modelId;
-        chatLaunchState = "launching";
 
         // Record the launch in recent models history
         recordRecentLaunch(modelId);
@@ -2550,10 +2547,12 @@
   ];
 
   // ── Seamless chat: launch models from chat view ──
-  type InFlightChatLaunchState = "launching" | "downloading" | "loading";
-  type ReadyLikeChatLaunchState = "idle" | "ready";
-  type ChatLaunchState = InFlightChatLaunchState | ReadyLikeChatLaunchState;
-
+  type ChatLaunchState =
+    | "idle"
+    | "launching"
+    | "downloading"
+    | "loading"
+    | "ready";
   let chatLaunchState = $state<ChatLaunchState>("idle");
   let pendingChatModelId = $state<string | null>(null);
   let selectedChatCategory = $state<string | null>(null);
@@ -3130,15 +3129,6 @@
     if (model) {
       pendingAutoMessage = { content, files };
       userForcedIdle = false;
-      // The selected model is already being placed or loaded; keep the queued
-      // message and let the existing launch state effects send it once ready.
-      if (
-        pendingChatModelId === model &&
-        chatLaunchState !== "idle" &&
-        chatLaunchState !== "ready"
-      ) {
-        return;
-      }
       launchModelForChat(model, "picker", messages().length > 0);
       return;
     }
@@ -4613,7 +4603,7 @@
                 type="button"
                 onclick={() => {
                   completeOnboarding();
-                  handleChatSend(chip);
+                  sendMessage(chip, undefined, thinkingEnabled());
                 }}
                 class="px-4 py-2 rounded-full border border-white/10 bg-white/5 text-sm text-white/60 hover:bg-white/10 hover:text-white/80 hover:border-white/20 transition-all duration-200 cursor-pointer"
               >
@@ -6110,7 +6100,7 @@
                           onclick={() => {
                             chatLaunchState = "idle";
                             selectedChatCategory = null;
-                            handleChatSend(prompt);
+                            sendMessage(prompt, undefined, thinkingEnabled());
                           }}
                           class="text-left px-3 py-2.5 text-xs text-exo-light-gray hover:text-white font-mono rounded-lg border border-exo-medium-gray/30 hover:border-exo-yellow/30 bg-exo-dark-gray/30 hover:bg-exo-dark-gray/60 transition-all duration-200 cursor-pointer"
                         >

docs/api.md

@@ -66,9 +66,7 @@ Creates a new model instance in the cluster.
 ```
 
 **Response:**
-Command acknowledgement. Instance creation is asynchronous; clients should wait
-for the model to appear through `/instance/await` before sending inference
-requests for that model.
+JSON description of the created instance.
 
 ### Delete Instance
 
@@ -96,31 +94,6 @@ Returns details of a specific instance.
 **Response:**
 JSON description of the instance.
 
-### Await Instance
-
-**GET** `/instance/await?model_id=...&timeout_seconds=0`
-
-Waits until API state contains an instance for the requested model. The response
-is an SSE stream so clients receive keep-alive comments while waiting.
-
-**Query parameters:**
-
-* `model_id`: string, required
-* `timeout_seconds`: float, optional, default `0`. `0` waits indefinitely;
-  positive values time out after that many seconds. Maximum positive value:
-  `300`.
-
-**Stream messages:**
-
-```text
-data: {"type": "ready", "instance": {...}}
-
-data: {"type": "timeout", "message": "No instance found for model ..."}
-```
-
-The HTTP status is `200` for both messages because the stream starts before the
-final result is known. The `type` field disambiguates the terminal message.
-
 ### Preview Placements
 
 **GET** `/instance/previews?model_id=...`
@@ -150,18 +123,17 @@ Computes a placement for a potential instance without creating it.
 **Response:**
 JSON object describing the proposed placement / instance configuration.
 
-### Place Instance
+### Place Instance (Dry Operation)
 
 **POST** `/place_instance`
 
-Places an instance for a model using the server's placement logic.
+Performs a placement operation for an instance (planning step), without necessarily creating it.
 
 **Request body:**
 JSON describing the instance to be placed.
 
 **Response:**
-Command acknowledgement. The instance may not be ready immediately; wait for it
-to appear through `/instance/await` before sending inference requests.
+Placement result.
 
 ## 3. Models
 
@@ -667,11 +639,10 @@ GET     /events
 
 # Instance Management
 POST    /instance
-GET     /instance/await
-GET     /instance/previews
-GET     /instance/placement
 GET     /instance/{instance_id}
 DELETE  /instance/{instance_id}
+GET     /instance/previews
+GET     /instance/placement
 POST    /place_instance
 
 # Models

docs/architecture.md

@@ -0,0 +1,84 @@
+# EXO Architecture overview
+
+EXO uses an _Event Sourcing_ architecture, and Erlang-style _message passing_. To facilitate this, we've written a channel library extending anyio channels with inspiration from tokio::sync::mpsc. 
+
+Each logical module - designed to be functional independently of the others - communicates with the rest of the system by sending messages on topics.
+
+## Systems
+
+There are currently 5 major systems:
+
+- Master
+    
+    Executes placement and orders events through a single writer
+
+- Worker
+    
+    Schedules work on a node, gathers system information, etc.#
+
+- Runner
+    
+    Executes inference jobs (for now) in an isolated process from the worker for fault-tolerance.
+
+- API
+    
+    Runs a python webserver for exposing state and commands to client applications
+
+- Election
+    
+    Implements a distributed algorithm for master election in unstable networking conditions
+
+## API Layer
+
+The API system uses multiple adapters to support multiple API formats, converting them to a single request / response type.
+
+### Adapter Pattern
+
+Adapters convert between external API formats and EXO's internal types:
+
+```
+Chat Completions → [adapter] → TextGenerationTaskParams → Application
+Claude Messages  → [adapter] → TextGenerationTaskParams → Application
+Responses API    → [adapter] → TextGenerationTaskParams → Application
+Ollama API       → [adapter] → TextGenerationTaskParams → Application
+```
+
+Each adapter implements two key functions:
+1. **Request conversion**: Converts API-specific requests to `TextGenerationTaskParams`
+2. **Response generation**: Converts internal `TokenChunk` streams back to API-specific formats (streaming and non-streaming)
+
+
+## Topics
+
+There are currently 5 topics:
+
+- Commands
+
+    The API and Worker instruct the master when the event log isn't sufficient. Namely placement and catchup requests go through Commands atm.
+
+- Local Events
+
+    All nodes write events here, the master reads those events and orders them
+
+- Global Events
+
+    The master writes events here, all nodes read from this topic and fold the produced events into their `State`
+
+- Election Messages
+
+    Before establishing a cluster, nodes communicate here to negotiate a master node.
+
+- Connection Messages
+
+    The networking system write mdns-discovered hardware connections here.
+
+
+## Event Sourcing
+
+Lots has been written about event sourcing, but it lets us centralize faulty connections and message ACKing with the following model.
+
+Whenever a device produces side effects, it captures those side effects in an `Event`. `Event`s are then "applied" to their model of `State`, which is globally distributed across the cluster. Whenever a command is received, it is combined with state to produce side effects, captured in yet more events. The rule of thumb is "`Event`s are past tense, `Command`s are imperative". Telling a node to perform some action like "place this model" or "Give me a copy of the event log" is represented by a command (The worker's `Task`s are also commands), while "this node is using 300GB of ram" is an event. Notably, `Event`s SHOULD never cause side effects on their own. There are a few exceptions to this, we're working out the specifics of generalizing the distributed event sourcing model to make it better suit our needs
+
+## Purity
+
+A significant goal of the current design is to make data flow explicit. Classes should either represent simple data (`FrozenModel`s typically, and `TaggedModel`s for unions) or active `System`s (Erlang `Actor`s), with all transformations of that data being "referentially transparent" - destructure and construct new data, don't mutate in place. We have had varying degrees of success with this, and are still exploring where purity makes sense.

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "crane": {
       "locked": {
-        "lastModified": 1775790182,
-        "narHash": "sha256-pG2RWVQY0Pe+rmmXJx+Jpyi+JcgjWzS18m7fcD1B64Q=",
+        "lastModified": 1779130139,
+        "narHash": "sha256-BLrtr42azquO7MdGFU5a7KiMl3YpFlTeIXqy1fT5GlQ=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "534982f1c41834b101e381b07b1121a4f065a374",
+        "rev": "edb38893982a3338972bb4a2ec7ce7c29ba10fd9",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1777708550,
-        "narHash": "sha256-Qif3UXT0l5OQq8H9pRWt4/ia4gF48MWK2oHKL8uVx8U=",
+        "lastModified": 1779185128,
+        "narHash": "sha256-Kl2bkmwZJD3n2KWDxuIlturZ7emqRK+anpD1LmDwpmY=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "74c1591efaff494756b8d35ebe357c6c2bbdca96",
+        "rev": "b7bd9323fe26a3b4f4bddbb2c2a1dacabced2f88",
         "type": "github"
       },
       "original": {
@@ -83,11 +83,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1775087534,
-        "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=",
+        "lastModified": 1778716662,
+        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b",
+        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
         "type": "github"
       },
       "original": {
@@ -118,11 +118,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1775595990,
-        "narHash": "sha256-OEf7YqhF9IjJFYZJyuhAypgU+VsRB5lD4DuiMws5Ltc=",
+        "lastModified": 1779102034,
+        "narHash": "sha256-vZJZjLo513IeI8hjzHFc6TDezUd4uCE2Eq4SNO3DNNg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4e92bbcdb030f3b4782be4751dc08e6b6cb6ccf2",
+        "rev": "687f05a9184cad4eaf905c48b63649e3a86f5433",
         "type": "github"
       },
       "original": {
@@ -168,11 +168,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1773870109,
-        "narHash": "sha256-ZoTdqZP03DcdoyxvpFHCAek4bkPUTUPUF3oCCgc3dP4=",
+        "lastModified": 1776659114,
+        "narHash": "sha256-qapCOQmR++yZSY43dzrp3wCrkOTLpod+ONtJWBk6iKU=",
         "owner": "pyproject-nix",
         "repo": "build-system-pkgs",
-        "rev": "b6e74f433b02fa4b8a7965ee24680f4867e2926f",
+        "rev": "ffaa2161dd5d63e0e94591f86b54fc239660fb2e",
         "type": "github"
       },
       "original": {
@@ -188,11 +188,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1775439158,
-        "narHash": "sha256-NHY9SJNU019n+8NCabBDtmuzRFeE2gZlYKHowp9bV24=",
+        "lastModified": 1778901413,
+        "narHash": "sha256-GSKXTAnFqRAMlZkJrIPcQMYf+lpMr66K3i60mB9STvc=",
         "owner": "pyproject-nix",
         "repo": "pyproject.nix",
-        "rev": "fb6b728260f3f32761367e9fd1e1a25b4245bcd0",
+        "rev": "a228447c3e179d477c1b6246ef3efa8cfe3c469a",
         "type": "github"
       },
       "original": {
@@ -218,11 +218,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1777639980,
-        "narHash": "sha256-6d7Hdurvbjc5uwJuc0YiK7rZBGj6Gs3uzfBFcTs+xCc=",
+        "lastModified": 1779074864,
+        "narHash": "sha256-0M3WqsWmtXmv9Ev/vnFfCHosWvISDwiuuhQ104UO3CI=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "64cdaeb06f69b6b769a492edd88b022ae88e8ca2",
+        "rev": "cdfe408d4b436e806ff525cb3e67588a6a009ed1",
         "type": "github"
       },
       "original": {
@@ -284,11 +284,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1775706324,
-        "narHash": "sha256-BTb4sydzX2B5/oNbvCdQFeSbk97xEnbb8bk84CiKCOs=",
+        "lastModified": 1778664018,
+        "narHash": "sha256-ogNyNANNLo0SMFevIeUpbTMOL9uUDu/hXvp7JlOYbwQ=",
         "owner": "pyproject-nix",
         "repo": "uv2nix",
-        "rev": "5707df99097375896a3dda811d492a2fabe63500",
+        "rev": "b48abe99ef639cd100c224898529370e5d935294",
         "type": "github"
       },
       "original": {

flake.nix

@@ -110,7 +110,7 @@
               nixpkgs-fmt.enable = true;
               ruff-format = {
                 enable = true;
-                excludes = [ "rust/exo_rs/exo_rs.pyi" ];
+                excludes = [ "rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi" ];
               };
               rustfmt = {
                 enable = true;
@@ -132,6 +132,7 @@
 
           packages = {
             default = self'.packages.exo;
+            babeld = pkgs.callPackage ./nix/babeld.nix { };
           } //
           lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin {
             metal-toolchain = pkgs.callPackage ./nix/metal-toolchain.nix { };

justfile

@@ -16,14 +16,14 @@ check:
     uv run basedpyright --project pyproject.toml
 
 sync:
-    uv sync --all-packages --extra mlx
+    uv sync --all-packages
 
 sync-clean:
-    uv sync --all-packages --extra mlx --force-reinstall --no-cache
+    uv sync --all-packages --force-reinstall --no-cache
 
 rust-rebuild:
     PYO3_PYTHON="$(uv run python -c 'import sys; print(sys.executable)')" cargo run --bin stub_gen
-    uv sync --reinstall-package exo_rs
+    uv sync --reinstall-package exo_pyo3_bindings
 
 build-dashboard:
     #!/usr/bin/env bash
@@ -37,7 +37,7 @@ package: build-dashboard
     rm -rf build
 
 build-app: rust-rebuild sync-clean package
-    env -u LD xcodebuild build -project app/EXO/EXO.xcodeproj -scheme EXO -configuration Debug -derivedDataPath app/EXO/build
+    xcodebuild build -project app/EXO/EXO.xcodeproj -scheme EXO -configuration Debug -derivedDataPath app/EXO/build
     @echo "\nBuild complete. Run with:\n  open {{justfile_directory()}}/app/EXO/build/Build/Products/Debug/EXO.app"
 
 clean:

nix/babeld.nix

@@ -0,0 +1,26 @@
+{ stdenv
+, lib
+, fetchgit
+}:
+stdenv.mkDerivation {
+  pname = "babeld";
+  version = "1.13.2-rc";
+
+  # TODO: pin to specific version/revision, or better yet, use a patch file
+  src = fetchgit {
+    url = "https://github.com/AndreiCravtov/babeld.git";
+    fetchSubmodules = true;
+    sha256 = "sha256-/qsoMSRhtwa/2hvACtFwbl+563o+TKxWMS684D+g8mk=";
+  };
+
+  outputs = [
+    "out"
+    "man"
+  ];
+
+  makeFlags = [
+    "PREFIX=${placeholder "out"}"
+    "ETCDIR=${placeholder "out"}/etc"
+  ];
+}
+

package-lock.json

@@ -1,6 +0,0 @@
-{
-  "name": "exo",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {}
-}

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
   "huggingface-hub>=1.8.0",
   "psutil>=7.0.0",
   "loguru>=0.7.3",
-  "exo-rs",                          # rust bindings
+  "exo-pyo3-bindings",               # rust bindings
   "anyio==4.11.0",
   "tiktoken>=0.12.0",                # required for kimi k2 tokenizer
   "hypercorn>=0.18.0",
@@ -26,7 +26,6 @@ dependencies = [
   "msgspec>=0.19.0",
   "zstandard>=0.23.0",
   "transformers>=5.6.2",
-  "python-daemon>=3.1.2",
 ]
 
 [project.scripts]
@@ -76,14 +75,22 @@ mlx-cuda13 = [
 ###
 
 [tool.uv.workspace]
-members = ["rust/exo_rs", "bench", "tools"]
+members = ["rust/exo_pyo3_bindings", "bench", "tools"]
 
 [tool.uv.sources]
-exo-rs = { workspace = true }
+exo-pyo3-bindings = { workspace = true }
 mlx = [
   { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" },
   { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx-0.32.0-cp313-cp313-manylinux_2_35_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" },
   { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx-0.32.0-cp313-cp313-manylinux_2_35_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine != 'aarch64'" },
+
+]
+mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/deepseek-v4" }
+mflux = { git = "https://github.com/evanev7/mflux", branch = "exo2" }
+torch = [
+  { index = "pytorch-cpu", marker = "sys_platform == 'linux' and extra == 'mlx-cpu' and extra != 'mlx-cuda13' and extra != 'mlx-cuda12'" },
+  { index = "pytorch-cu128", marker = "sys_platform == 'linux' and extra == 'mlx-cuda12' and extra != 'mlx-cuda13' " },
+  { index = "pytorch-cu130", marker = "sys_platform == 'linux' and extra == 'mlx-cuda13'" },
 ]
 mlx-cuda-12 = [
   { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx_cuda_12-0.32.0-py3-none-manylinux_2_35_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" },
@@ -93,13 +100,6 @@ mlx-cuda-13 = [
   { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx_cuda_13-0.32.0-py3-none-manylinux_2_35_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64'" },
   { url = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv/releases/download/mlx_cuda/mlx_cuda_13-0.32.0-py3-none-manylinux_2_35_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine != 'aarch64'" },
 ]
-mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/deepseek-v4" }
-mflux = { git = "https://github.com/evanev7/mflux", branch = "exo2" }
-torch = [
-  { index = "pytorch-cpu", marker = "sys_platform == 'linux' and extra == 'mlx-cpu' and extra != 'mlx-cuda13' and extra != 'mlx-cuda12'" },
-  { index = "pytorch-cu128", marker = "sys_platform == 'linux' and extra == 'mlx-cuda12' and extra != 'mlx-cuda13' " },
-  { index = "pytorch-cu130", marker = "sys_platform == 'linux' and extra == 'mlx-cuda13'" },
-]
 torchvision = [
   { index = "pytorch-cpu", marker = "sys_platform == 'linux' and extra == 'mlx-cpu' and extra != 'mlx-cuda13' and extra != 'mlx-cuda12'" },
   { index = "pytorch-cu128", marker = "sys_platform == 'linux' and extra == 'mlx-cuda12' and extra != 'mlx-cuda13'" },
@@ -240,7 +240,7 @@ torchaudio = ["torch"]
 ###
 
 [tool.ruff]
-extend-exclude = [".typings/**", "rust/exo_rs/**", "bench/vendor/**"]
+extend-exclude = [".typings/**", "rust/exo_pyo3_bindings/**", "bench/vendor/**"]
 
 [tool.ruff.lint]
 extend-select = ["I", "N", "B", "A", "PIE", "SIM"]

python/parts.nix

@@ -44,21 +44,20 @@ let
         paths = builtins.concatMap (p: [ (lib.getBin p) (lib.getLib p) (lib.getDev p) ]) (cudaLibs ++ [ cudaPackages.cuda_nvcc cuda_cccl_compat ]);
       };
       exoOverlay = final: prev: {
-        # Replace workspace exo_rs with Nix-built wheel.
+        # Replace workspace exo_pyo3_bindings with Nix-built wheel.
         # Preserve passthru so mkVirtualEnv can resolve dependency groups.
         # Copy .pyi stub + py.typed marker so basedpyright can find the types.
-        exo-rs = pkgs.stdenv.mkDerivation {
-          pname = "exo-rs";
+        exo-pyo3-bindings = pkgs.stdenv.mkDerivation {
+          pname = "exo-pyo3-bindings";
           version = "0.1.0";
-          src = self'.packages.exo-rs;
+          src = self'.packages.exo_pyo3_bindings;
           # Install from pre-built wheel
           nativeBuildInputs = [ final.pyprojectWheelHook ];
           dontStrip = true;
-          passthru = prev.exo-rs.passthru or { };
-
+          passthru = prev.exo-pyo3-bindings.passthru or { };
           postInstall = ''
-            local siteDir=$out/${final.python.sitePackages}/exo_rs
-            cp ${inputs.self}/rust/exo_rs/exo_rs.pyi $siteDir/
+            local siteDir=$out/${final.python.sitePackages}/exo_pyo3_bindings
+            cp ${inputs.self}/rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi $siteDir/
             touch $siteDir/py.typed
           '';
         };
@@ -224,7 +223,7 @@ let
           ++ lib.optionals isDarwin [ pkgs.macmon ];
           passthru = {
             venv = venv name;
-            evenv = ((pythonSet.overrideScope editableOverlay).mkVirtualEnv "${name}-evenv" (members // { exo = (members.exo or [ ]) ++ [ "dev" ]; exo-rs = [ ]; })).overrideAttrs (_: {
+            evenv = ((pythonSet.overrideScope editableOverlay).mkVirtualEnv "${name}-evenv" (members // { exo = (members.exo or [ ]) ++ [ "dev" ]; })).overrideAttrs (_: {
               venvSkip = [ "lib/python${python.pythonVersion}/site-packages/mlx/share/cmake/*" "lib/python${python.pythonVersion}/site-packages/build_backend.py" ];
             });
           } // lib.optionalAttrs cudaSupport {

resources/inference_model_cards/moonshotai--Kimi-K2.7-Code.toml

@@ -1,36 +0,0 @@
-model_id = "moonshotai/Kimi-K2.7-Code"
-n_layers = 61
-hidden_size = 7168
-num_key_value_heads = 64
-supports_tensor = true
-tasks = ["TextGeneration"]
-family = "kimi"
-quantization = ""
-base_model = "Kimi K2.7 Code"
-capabilities = ["text", "thinking", "thinking_toggle", "vision"]
-
-context_length = 262144
-backends = ["MlxMetal", "MlxCuda", "MlxCpu"]
-[storage_size]
-in_bytes = 595204986173
-
-# Vision tower + mm_projector extracted unmodified (bf16) from the official
-# repo, in the same format as exolabs/Kimi-K2.6-vision; extraction script
-# included in the weights repo. Vision config is identical to Kimi-K2.6's.
-[vision]
-image_token_id = 163605
-model_type = "kimi_vl"
-weights_repo = "aidiffuser/Kimi-K2.7-Code-vision"
-processor_repo = "moonshotai/Kimi-K2.7-Code"
-
-# Source: https://huggingface.co/moonshotai/Kimi-K2.7-Code
-# (recommends temperature 1.0 / top_p 0.95 for thinking mode, same as K2.6)
-[sampling_defaults]
-temperature = 1.0
-top_p = 0.95
-min_p = 0.01
-
-[sampling_defaults.non_thinking]
-temperature = 0.6
-top_p = 0.95
-min_p = 0.01

result

@@ -0,0 +1 @@
+/nix/store/41rd4g2p69a2106qscl3xvkzmh6j7nry-babblerd

rust/babblerd/.gitignore

@@ -0,0 +1 @@
+/pbprobe/*

rust/babblerd/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "babblerd"
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+color-eyre = "0.6.5"
+clap = { version = "4.5.53", features = ["derive"] }
+futures-lite.workspace = true
+ipnet = "2.12.0"
+nix = { version = "0.31", features = ["fs", "signal", "process", "user", "net", "uio"] }
+netdev = "0.42"
+ahash = "0.8.12"
+arrayvec = "0.7.6"
+crossbeam-channel = "0.5.15"
+hashbrown = "0.16.0"
+iroh-quinn-udp = { version = "0.8.0", default-features = false, features = ["fast-apple-datapath"] }
+libc = "0.2"
+mio = { version = "1.1.0", features = ["net", "os-ext", "os-poll"] }
+n0-watcher = "0.6"
+netwatch = "0.16"
+rand = "0.10"
+route_manager = "0.2.11"
+slab = "0.4.11"
+socket2 = "0.6.1"
+tokio = { workspace = true, features = ["full"] }
+tracing = "0.1.44"
+tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
+tun-rs = "2.8.1" # if you update, it causes transitive dependeny clashes that need patch.crates-io fixes or whatnot, too long to do now :)
+
+# parsing
+memchr = "2.8"
+winnow = "1.0"
+thiserror = "2.0"
+macaddr = "1.0"
+zerocopy = { version = "0.8.31", features = ["derive"] }
+
+[lints]
+workspace = true

rust/babblerd/examples/link_profile.rs

@@ -0,0 +1,4 @@
+fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
+    babblerd::profiling::standalone::run_from_env()
+}

rust/babblerd/examples/pbprobe_link.rs

@@ -0,0 +1,4 @@
+fn main() -> color_eyre::Result<()> {
+    color_eyre::install()?;
+    babblerd::profiling::pbprobe::standalone::run_from_env()
+}

rust/babblerd/future_architectural_directions.md

@@ -0,0 +1,391 @@
+# `babblerd` Future Architectural Directions
+
+This file is not a debt list.
+
+Use [shortcuts.md](./shortcuts.md) for concrete shortcuts, footguns, and
+implementation compromises that should be cleaned up later.
+
+This file is for directional reasoning:
+
+- what the current architecture is trying to become,
+- which major steps are worth doing next,
+- and why those steps are ordered the way they are.
+
+It should evolve as the architecture evolves.
+
+## Current Position
+
+`babblerd` is no longer just a thin wrapper around `babeld`.
+
+It now has the beginnings of a real daemon architecture:
+
+- a resident daemon process,
+- a resident TUN interface,
+- a keepalive-driven daemon core,
+- a heavy routing stack that can turn on and off,
+- a typed Babel control/runtime layer,
+- a derived FIB layer,
+- a dedicated dataplane thread wired into the routing stack,
+- a persisted node identity,
+- and a central config module.
+
+That is enough structure to stop treating the whole project as “just Babel
+plumbing”.
+
+For bring-up, the current tree also contains a temporary internal self-client
+that connects to the public socket and periodically sends keepalive commands.
+That is only a testing scaffold so the routing stack stays on without a real
+frontend process yet. It should be removed once a real controller exists.
+
+It is also enough structure to begin building the actual dataplane without
+needing to perfect every IPC and control-plane detail first.
+
+Stable one-hop forwarding is now proven on the four-Mac Thunderbolt lab for an
+adjacent pair after switching dataplane TUN I/O on macOS to `tun-rs`
+`SyncDevice::recv`/`send` instead of raw fd `read`/`write`.
+
+Steady-state ICMPv6 reachability is also now green across the full four-node
+lab ring, and small generic TCP application payloads now work too once the
+mesh has converged. The current remaining gap is no longer basic correctness of
+non-ICMP transport traffic, but sustained throughput under load: current
+`iperf3` testing transfers an initial burst and then collapses into heavy
+retransmits and near-zero receive-side throughput.
+
+So the current architecture is good enough for continued correctness and
+reliability bring-up, but serious performance work should wait until that
+throughput-collapse behavior is understood.
+
+## The Most Important Architectural Decision
+
+The current codebase is already good enough to serve as the shell around a
+first real dataplane.
+
+That means the next major effort should **not** automatically be:
+
+- replacing the line protocol with `zbus`,
+- perfecting lease ownership,
+- or fully polishing lifecycle semantics.
+
+Those are still desirable, but they are not the blocking step for getting to a
+working end-to-end system.
+
+The next big milestone should be:
+
+- a real UDP dataplane,
+- driven by the current daemon core and current Babel-derived state,
+- with a basic but coherent forwarding model.
+
+In other words: move from “control plane with architecture” to “working router
+with acceptable architecture”.
+
+## Near-Term Goal
+
+The near-term target is:
+
+> a basic end-to-end MVC where:
+>
+> - the daemon has a stable node identity,
+> - the daemon can be kept alive by the frontend,
+> - the daemon maintains Babel-derived routing state,
+> - the daemon can forward packets through a UDP overlay between nodes,
+> - and the frontend can inspect enough daemon state to be useful.
+
+This does **not** require the final IPC architecture first.
+
+## Recommended Next Phase
+
+### 1. Validate and harden the first UDP dataplane
+
+This should be the next major feature.
+
+The basic pieces are now in place:
+
+- `fib.rs` derives immutable forwarding snapshots from `BabelState`,
+- those snapshots now carry the admitted interface set as well as routes,
+- `dataplane.rs` provides a dedicated-thread hot-path module using `mio`,
+  `socket2`, `crossbeam-channel`, `hashbrown`, `ahash`, `slab`, and
+  `arrayvec`.
+- `routing_stack.rs` now starts the dataplane and publishes coalesced
+  `FibSnapshot` updates into it.
+- dataplane socket ownership is now driven by interfaces that currently have
+  live Babel neighbours rather than only by currently selected routes, and
+  retained sockets are refreshed when an `ifname` resolves to a new ifindex.
+- socket reconcile is now best-effort under interface churn: transient
+  resolution/open failures are logged and retried without killing the
+  dataplane during reconcile.
+- unchanged FIB snapshots are now deduplicated in the control plane, so the
+  dataplane also carries its own lightweight timer-driven reconcile retry for
+  admitted interfaces that still do not have usable sockets.
+- the stable-link packet path is now working on the lab ring for adjacent
+  one-hop traffic.
+
+So the next step is no longer “invent or wire the modules”.
+
+It is:
+
+- extend live validation from adjacent one-hop traffic to multi-hop and churn,
+- harden the remaining dataplane failure/reporting behavior under interface
+  churn,
+- confirm the interface-bound UDP socket model behaves correctly on the target
+  machines,
+- and then fill the first obvious protocol gaps such as ICMPv6 error handling.
+
+One caveat that is now proven on the lab Macs: the current macOS receive path
+cannot treat "which UDP socket got the packet" as trustworthy interface
+attribution. In live tests, packets sent directly over one Thunderbolt
+interface are still being received by a different reuseport socket while the
+peer scope-id reflects the real physical ingress interface. That means the
+current multi-socket receive model is acceptable for basic forwarding bring-up,
+but it is not yet a reliable source of receive-side interface truth on macOS.
+
+The likely long-term fix is to move receive-side interface attribution onto
+ancillary packet metadata (`IPV6_PKTINFO` / receive-interface data) rather than
+inferring it from which socket woke up.
+
+For the current four-Mac Thunderbolt lab, the broad macOS `en*` watcher
+heuristic has proven too permissive in practice. The dataplane now corrects
+that somewhat by only owning sockets on interfaces that Babel has actually
+formed neighbour adjacencies on, but the watcher/bootstrap side is still broad
+and may still need a per-host allowlist during bring-up while the long-term
+admission policy is refined.
+
+The current broad-admission behavior is acceptable for v1 as long as point to
+point and multihop forwarding remain reliable, but it does mean that multiple
+wired interfaces can become equally admissible at once.
+
+The desired longer-term policy is:
+
+- admit any interface that Babel can actually form a live neighbour adjacency
+  on, regardless of naming convention,
+- keep that broad admissibility for reachability,
+- but rank competing links by measured quality rather than treating all wired
+  links as equivalent.
+
+That future link-scoring direction likely requires:
+
+- computing local link metrics such as latency, loss, and possibly sustainable
+  throughput without generating excessive probe traffic,
+- sharing or projecting those metrics into the distributed routing view in a
+  way Babel can actually consume,
+- and then teaching the Babel path-selection logic to prefer the better direct
+  link when multiple usable adjacencies exist.
+
+That is explicitly post-v1 work. For the first version, correctness and
+reliability of the multihop mesh matter more than optimal link preference.
+
+The current sustained-throughput investigation is therefore focused on two
+nearer-term issues before any serious performance tuning:
+
+- understanding whether load-induced loss is primarily backpressure on UDP
+  socket send / TUN reinjection,
+- and separating that from the now-proven macOS receive-side interface
+  attribution oddities.
+
+The live restart-sensitive failures are now narrowed more precisely than that.
+On the four-Mac lab, a restarted node can receive an encapsulated packet, push
+it through local TUN delivery successfully, and still blackhole the exchange
+because the generated return packet is resolved onto a worse broad-admission
+path (for example `en1`) instead of the direct Thunderbolt neighbour that just
+delivered the request.
+
+So the current main blocker for reliable restart/churn behavior is not "the
+dataplane cannot receive or decapsulate packets". It is "the control plane can
+still choose an asymmetric installed route that is valid enough for Babel to
+advertise, but poor enough to break or destabilize the actual return path".
+
+That strengthens the case for the planned future link-quality policy:
+
+- broad admissibility is still the right v1 reachability rule,
+- but equally admissible wired links need a better ranking signal than today's
+  flat wired costs,
+- otherwise restart-time route selection can still land on a functionally worse
+  path even when a direct high-quality neighbour exists.
+
+The current tree now owns the kernel route that steers overlay traffic into the
+resident TUN interface:
+
+- the local node `/128` address is installed on the TUN device,
+- `EXO_ULA_PREFIX -> tunX` is added when the routing stack turns on,
+- that prefix route is removed when the routing stack turns off,
+- and `babeld` kernel installs remain disabled.
+
+That means local application traffic can now be steered into the overlay once
+the UDP dataplane is active.
+
+The first version can stay simple:
+
+- one UDP datagram carries exactly one inner IPv6 packet,
+- no custom framing,
+- no batching,
+- no crypto,
+- no relays,
+- no multiplexed control/data protocol.
+
+The dataplane should:
+
+- read packets from TUN,
+- classify local-delivery vs forwarding,
+- look up next-hop information from a derived forwarding view,
+- send encapsulated packets to direct neighbors over UDP,
+- receive UDP packets from neighbors,
+- decapsulate them,
+- either inject them locally into TUN or forward them onward.
+
+This gives the project a real “V” and “M” to go with the current daemon/control
+shell.
+
+The current tree now hardcodes:
+
+- physical link MTU assumption: `1500`
+- outer overhead assumption: `40 bytes IPv6 + 8 bytes UDP`
+- derived TUN MTU: `1452`
+
+That is acceptable for bring-up, but it is still only a temporary model.
+
+The future direction should be:
+
+- route-aware MTU derivation,
+- PMTUD-aware behavior,
+- and better support for environments where hop-to-hop links can use jumbo
+  frames without exposing that complexity to user traffic.
+
+### 2. Keep the derived forwarding table separate from `BabelState`
+
+`BabelState` should remain a mirror of what `babeld` says.
+
+The current code now reflects that direction:
+
+- `BabelState` is still the protocol mirror,
+- `FibSnapshot` is the dataplane view.
+
+The next layer should be a derived forwarding table/FIB that:
+
+- is keyed by destination prefix or node address,
+- only keeps the routes the dataplane should actually use,
+- captures next hop / outgoing interface / any other forwarding metadata,
+- and is cheap for the dataplane to consult.
+
+This avoids mixing:
+
+- “what Babel currently knows”
+- with
+- “what the UDP router should do with packets”.
+
+### 3. Add a stronger public state/readiness model
+
+The current `ServiceState` is useful, but it is only lifecycle state:
+
+- `Off`
+- `Starting`
+- `On`
+- `Stopping`
+
+That is not the same thing as routing readiness.
+
+Once the dataplane exists, a separate readiness/status view should exist too.
+For example, the frontend may want to distinguish:
+
+- daemon is idle,
+- daemon is starting,
+- Babel is running but no eligible interfaces exist,
+- interfaces exist but no neighbors are usable,
+- forwarding is nominal,
+- forwarding is degraded.
+
+That should be modeled separately from `ServiceState`, not by making
+`ServiceState::On` carry too much meaning.
+
+## What Can Wait Until After the Dataplane Exists
+
+These are still desirable, but they do not need to block the first end-to-end
+router:
+
+### `zbus` / D-Bus-style IPC
+
+This is still the likely long-term direction.
+
+But the current line protocol is good enough for:
+
+- `keepalive <ttl_ms>`
+- `get-state`
+
+while the dataplane is being built.
+
+So `zbus` should remain a planned improvement, not the immediate blocker.
+
+### Per-client leases
+
+The daemon should eventually track leases per client/connection rather than via
+a single global keepalive deadline.
+
+That is a real architectural improvement, but it is control-plane polish rather
+than dataplane unblocker.
+
+It can happen after the first router path works.
+
+### Structured diagnostics/debug output
+
+Right now diagnostics are tracing-only.
+
+That is acceptable for development while the dataplane is first being brought
+up.
+
+A configurable debug stream or structured diagnostics feed should be added
+later, preferably once the public IPC shape is stabilized.
+
+## The First Dataplane Should Stay Intentionally Small
+
+The first version should avoid solving every future overlay concern.
+
+It should **not** attempt to solve:
+
+- encryption,
+- authentication,
+- path quality metrics beyond what Babel already provides,
+- batching,
+- relay protocols,
+- or multi-transport negotiation.
+
+The first version should prove the simplest useful thing:
+
+- stable node addresses,
+- UDP transport between neighbors,
+- Babel-driven next-hop selection,
+- TUN injection/extraction,
+- packet forwarding that actually works end-to-end.
+
+If that works, the rest can be improved incrementally.
+
+## Architectural Path After the First Dataplane Works
+
+Once the basic dataplane exists and works, the likely next path is:
+
+1. Improve the public state/readiness model.
+2. Replace the ad-hoc control socket with `zbus`.
+3. Replace the single keepalive deadline with per-client leases.
+4. Tighten interface admission beyond the current broad heuristic.
+5. Pin and explicitly invoke the exact forked `babeld`.
+6. Harden node-id file mode checks and other local security edges.
+7. Revisit diagnostics streaming.
+8. Revisit platform abstractions around TUN / transport / forwarding.
+
+That ordering is intentional:
+
+- prove the router first,
+- then harden and refine the daemon architecture around it.
+
+## Guiding Principle
+
+The project should prefer:
+
+- a coherent working router with a few acknowledged shortcuts
+
+over:
+
+- a beautifully abstract control plane that still does not move packets.
+
+That does **not** mean ignoring architecture.
+
+It means using the current architecture as a platform for the next real
+capability, rather than repeatedly polishing the control shell before the
+dataplane exists.

rust/babblerd/handoff.md

@@ -0,0 +1,352 @@
+# babblerd Handoff
+
+This is the current handoff for a new session picking up `babblerd` work.
+
+## Repo / Branch / State
+
+- Repo: `/home/royalguard/Desktop/exo-all/exo`
+- Branch: `babbler`
+- Current HEAD: `b0f508ac` (`fix babbler ula prefix`)
+- Recent relevant commits:
+  - `b0f508ac` fix babbler ula prefix
+  - `2e17eb03` retry skipped dataplane sockets
+  - `4efc883f` dedup unchanged fib snapshots
+  - `79a5a84b` instrument dataplane backpressure
+  - `f0140a81` trim dataplane logs for live testing
+  - `df2760b6` admit dataplane interfaces from babel neighbours
+  - `e1e4643e` use tun-rs packet io in dataplane
+- Current working tree state when this was written:
+  - modified: `rust/babblerd/shortcuts.md`
+  - modified: `rust/babblerd/future_architectural_directions.md`
+  - untracked: `.codex`
+- Local checks pass:
+  - `cargo fmt -p babblerd`
+  - `cargo check -p babblerd`
+  - `cargo test -p babblerd`
+
+## Core Conclusion
+
+The project is past the “is the overlay architecture wrong?” phase.
+
+The current architecture is the right one:
+
+- `babeld` is control plane only
+- Babel kernel installs are disabled
+- local mesh traffic is steered into a resident TUN
+- userspace dataplane forwards one inner IPv6 packet per UDP datagram hop-by-hop over neighbour link-locals
+
+So the main remaining work is now:
+
+- route-selection debugging
+- restart/convergence robustness
+- load/backpressure/throughput behavior
+- eventually link scoring across multiple admissible wired links
+
+## Why The Old Approach Failed
+
+The original macOS idea was effectively:
+
+- let `babeld` install routes
+- try to make kernel source selection behave
+
+That did not work cleanly for this use case:
+
+- no usable IPv6 pref-src route install path on macOS/BSD for this design
+- no native source-specific IPv6 routing model that solves the app behavior wanted here
+- putting ULAs on `lo0` or `utun` did not reliably fix source selection
+- app-aware binding alone was not enough in practice
+
+That is why the design pivoted to the userspace overlay dataplane.
+
+## Files To Read First
+
+- `src/daemon.rs`
+- `src/routing_stack.rs`
+- `src/dataplane.rs`
+- `src/fib.rs`
+- `src/tun.rs`
+- `src/babel/runtime.rs`
+- `src/route_ctl.rs`
+- `lab_topology_reference.md`
+- `shortcuts.md`
+- `future_architectural_directions.md`
+
+## Current Intended Architecture
+
+Model:
+
+- one stable node `/128` on TUN
+- `EXO_ULA_PREFIX -> tunX` installed by `babblerd`
+- `babeld` kernel installs are disabled / ignored
+- `BabelState` mirrors `babeld`
+- `FibSnapshot` is a reduced immutable dataplane view
+- control plane stays on Tokio
+- dataplane is a dedicated thread
+- one UDP datagram carries exactly one inner IPv6 packet
+- no custom framing yet
+- outer IPv6 destination is neighbour link-local
+- outer UDP port is `router_udp_port`
+
+This is the v1 forwarding model:
+
+- exact-match `/128` host routes only
+- interface identity in FIB is `ifname`
+- dataplane owns sockets from admitted interface set
+- admitted dataplane interfaces come from live Babel neighbours
+
+## Important Design Decisions Already Landed
+
+- typed Babel parsing/state model, not raw string handling
+- monitor-driven Babel runtime, not periodic dump polling
+- persistent node identity across restarts
+- explicit daemon lifecycle: `Off | Starting | On | Stopping`
+- resident TUN lifetime, separate heavy routing stack
+- dataplane thread + immutable FIB snapshot swaps
+- socket ownership from admitted interface set, not only current route set
+- same-name/new-ifindex socket refresh handled
+- timer-driven socket reconcile retry in dataplane, so deduped unchanged FIB snapshots do not suppress retries forever
+- dataplane exit supervision back into routing stack / daemon
+- macOS dataplane uses `tun-rs` packet I/O (`SyncDevice::recv/send`), not raw fd reads/writes
+
+## Very Important Fix After Earlier Handovers
+
+The previously-deployed node addresses were wrong.
+
+There was a real bug in `EXO_ULA_PREFIX` construction:
+
+- intended prefix: `fde0:20c6:1fa7:ffff::/64`
+- broken runtime prefix had become: `20c6:1fa7:ffff:0::/64`
+
+Cause:
+
+- `config.rs` used a `u128` left-shift construction that dropped the high `fde0` bits
+
+Fix:
+
+- commit `b0f508ac` changed the prefix constant to explicit hextets and added a regression test
+
+Live verification after redeploy:
+
+- `e4 utun5`: `fde0:20c6:1fa7:ffff:cc78:aec2:d64e:f125/128`
+- `e2 utun5`: `fde0:20c6:1fa7:ffff:aeb:e53a:cb17:aa42/128`
+- `e11 utun5`: `fde0:20c6:1fa7:ffff:34a:26dd:46ff:1a3f/128`
+- `e16 utun5`: `fde0:20c6:1fa7:ffff:7c5d:5e2d:54df:e665/128`
+
+So any older notes mentioning the truncated non-ULA prefix are stale.
+
+## Current Dataplane Behavior
+
+In `src/dataplane.rs`:
+
+- TUN ingress:
+  - read inner IPv6 packet
+  - parse destination
+  - drop self-directed
+  - FIB lookup
+  - send raw inner packet as UDP payload to neighbour
+- UDP ingress:
+  - receive UDP payload
+  - payload is raw inner IPv6 packet
+  - if destination local, inject into TUN
+  - else decrement inner hop limit and forward
+
+Fast-path traits:
+
+- dedicated OS thread
+- `mio` polling
+- `socket2` UDP sockets
+- immutable FIB snapshot swaps over `crossbeam-channel`
+- no lock on packet lookup path
+
+## What Works
+
+These things are now real:
+
+- one-hop two-node `ping6`
+- adjacent dataplane path
+- small low-rate UDP matrix
+- encapsulation / decapsulation itself
+- basic generic TCP correctness after convergence
+
+A very important live proof point:
+
+On a failing restart-sensitive `e11 -> e16` case, the dataplane itself still did the right work:
+
+- `e11` emitted the encapsulated packet on `en3`
+- `e16` received that UDP packet on `en18`
+- `e16` dataplane delivered the inner packet into TUN
+- the local stack generated a reply packet
+
+So the dataplane is not fundamentally broken anymore.
+
+## What Is Still Broken
+
+The main remaining live problem is restart/convergence behavior and path selection quality.
+
+Observed failure shape:
+
+- after restart, a node can receive and decapsulate correctly
+- but the return packet gets resolved onto a worse path, often `en1`, instead of the direct Thunderbolt link
+- this creates blackholes or severe instability
+
+So the current blocker is:
+
+- control-plane / derived-FIB route choice under broad multi-link admission
+- not “UDP overlay cannot carry packets”
+
+## Very Important macOS Receive-Side Finding
+
+On macOS, receive-side socket attribution is not trustworthy in the current one-socket-per-interface model.
+
+Observed live behavior:
+
+- traffic sent directly over one Thunderbolt link can be delivered to a different UDP socket than expected
+- the peer scope-id still reflects the real ingress interface
+
+Implication:
+
+- do not trust “which socket woke up” as authoritative ingress truth on macOS
+- if receive-side interface attribution matters, use peer scope-id and likely ancillary packet metadata later
+
+This is a real quirk, but it is not the primary blocker for the current restart blackhole.
+
+## Key Local FIB Caveat
+
+Do not assume route-choice issues are only Babel’s fault.
+
+In `src/fib.rs`, `FibBuilder` collapses multiple installed host routes by choosing the lowest:
+
+- `metric`
+- then `refmetric`
+- then `handle`
+
+So if restart churn leaves multiple `installed=yes` candidates, babblerd’s derived `FibSnapshot` can still be part of why traffic goes via `en1`.
+
+That means the next debug pass must compare all three:
+
+1. raw Babel route events / dump
+2. current `BabelState`
+3. derived `FibSnapshot`
+
+Not Babel in isolation.
+
+## Lab Topology / Operations
+
+Source of truth file:
+
+- `lab_topology_reference.md`
+
+Key facts:
+
+- four Mac minis
+- hostnames:
+  - `e4@e4`
+  - `e2@e2`
+  - `e11@e11`
+  - `e16@e16`
+- ring topology:
+  - `e4 -> e2 -> e11 -> e16 -> e4`
+- remote repo path:
+  - `~/babeld-exo`
+- each remote must `git pull` before running
+- current start command:
+  - `cd ~/babeld-exo && git pull && RUST_LOG=info sudo -E nix run .#babblerd --impure`
+- temporary internal keepalive client exists, so no external `nc -U ...` client is needed just to keep daemon alive
+- remote `iperf3` exists at:
+  - `/opt/homebrew/bin/iperf3`
+
+## Current Docs Are Mostly Accurate
+
+Read:
+
+- `shortcuts.md`
+- `future_architectural_directions.md`
+
+They correctly capture:
+
+- broad admissibility is acceptable for v1 reachability
+- flat wired costs are not enough for good best-path choice
+- throughput under load is still bad
+- restart-sensitive failures are now dominated by route selection, not dataplane decode
+- there is still debt around interface identity, macOS receive attribution, IPC/authz, and incomplete ICMP/PMTUD behavior
+
+## Important Remaining Technical Debt
+
+Still unresolved:
+
+- public IPC socket is too open
+- `ServiceState::On` is not the same as “fully ready/routable”
+- broad interface admission is still heuristic
+- route ownership of `EXO_ULA_PREFIX` is aggressive
+- no ICMPv6 Time Exceeded
+- no Packet Too Big handling
+- no real backpressure/queueing; `WouldBlock` is still drop-on-backpressure
+- throughput collapses under sustained load
+- macOS receive-side interface attribution needs a better long-term path
+- multi-link path selection is still too naive
+
+## What Not To Revisit Right Now
+
+These are settled enough for now:
+
+- overlay architecture itself
+- TUN + userspace UDP forwarding model
+- one-packet-per-datagram framing
+- control plane on Tokio, dataplane on dedicated thread
+- exact-match `/128` FIB for v1
+- `tun-rs` packet I/O on macOS instead of raw fd reads/writes
+- disabling Babel kernel installs and owning `EXO_ULA_PREFIX -> tunX` locally
+
+## Best Next Debugging Step
+
+For the restart-sensitive `en1` misroute, inspect all three together for a single problematic `/128` pair:
+
+1. raw Babel route events over time
+2. current `BabelState`
+3. derived `FibSnapshot`
+
+Goal:
+
+- determine whether the bad path is already in Babel’s installed route set
+- or introduced when `FibBuilder` collapses multiple `installed=yes` routes
+
+If equal-cost multi-link route choice is the problem, add only one temporary v1 policy knob:
+
+- either Babel interface-cost bias
+- or local FIB depreference
+
+Do not do both at once.
+
+## Best Next Live Tests
+
+1. full directed `ping6` matrix on node `/128`s
+2. small directed UDP matrix
+3. short soak tests on adjacent and two-hop pairs
+4. restart/convergence tests
+5. physical churn tests
+6. for failures, always capture:
+   - symptom
+   - raw Babel route state / dump
+   - current `BabelState`
+   - derived FIB state if relevant
+   - dataplane logs
+   - relevant `ifconfig`
+
+## Short Version
+
+The project is now in the:
+
+- route-selection debugging
+- restart convergence
+- throughput / backpressure robustness
+
+phase.
+
+The dataplane is basically real.
+
+The current main question is not “can the overlay forward packets at all?”
+
+It is:
+
+- why do restart-time and multi-link route choices still select worse return paths
+- and whether that bad choice originates in Babel’s installed set or in local FIB collapse

rust/babblerd/lab_topology_reference.md

@@ -0,0 +1,52 @@
+# Lab Topology Reference
+
+This file records the current, still-relevant lab topology and bring-up
+context for `babblerd`.
+
+## Hosts
+
+- The lab consists of four Mac minis.
+- SSH targets:
+  - `e4@e4`
+  - `e2@e2`
+  - `e11@e11`
+  - `e16@e16`
+- You can SSH into these machines directly to inspect or run commands.
+
+## Physical Topology
+
+- The machines are connected in a Thunderbolt ring:
+  - `e4 -> e2 -> e11 -> e16 -> e4`
+- The Thunderbolt-facing interface names are not fixed to `en2` and `en3`.
+  macOS can expose additional Thunderbolt links as other `en*` interfaces such
+  as `en5`, `en6`, or host-specific names after reconfiguration.
+- Treat `en2,en3` as an old bring-up heuristic only. For normal lab testing,
+  run without `BABBLER_INTERFACE_ALLOWLIST` and let `babblerd`/Babel discover
+  the live interfaces.
+
+## Repository Location On The Macs
+
+- Each machine has a checkout of the Exo repository at:
+  - `~/babeld-exo`
+- That checkout is expected to already be on the correct branch for this work.
+
+## Running `babblerd`
+
+From `~/babeld-exo`, pull first so the machine is not testing stale commits,
+then start `babblerd`:
+
+```sh
+cd ~/babeld-exo
+git pull
+RUST_LOG=info sudo -E nix run .#babblerd --impure
+```
+
+If broad interface discovery causes unrelated links to interfere with a
+specific debug run, `BABBLER_INTERFACE_ALLOWLIST` is still available as a
+temporary escape hatch. Do not use it as the default lab topology description.
+
+## Important Current Note
+
+- With the current codebase, `babblerd` has an internal dummy keepalive client.
+- That means you do **not** need to connect an external client socket just to
+  make the daemon stay active during testing.

rust/babblerd/pbprobe_implementation_plan.md

@@ -0,0 +1,48 @@
+# PBProbe Implementation Plan
+
+This file tracks the staged implementation and validation of a paper-faithful
+PBProbe profiler for link-local lab links.
+
+## Stage 1: Local Implementation
+
+- Add `src/profiling/pbprobe/` as a separate module from the simple packet
+  train profiler.
+- Implement the paper protocol:
+  - START initiates one direction.
+  - RTS requests each sample.
+  - the sender replies with a packet bulk of length `k`, meaning `k + 1`
+    packets.
+  - the receiver measures first and last packet arrival time, delay sum, and
+    dispersion.
+  - END reports the selected sample and estimate.
+- Implement Algorithm 1:
+  - start with `k = 1`.
+  - if measured minimum dispersion is below `D_thresh`, multiply `k` by 10 and
+    restart.
+  - otherwise pace samples with `G = 2D / U`.
+  - stop after fixed `n` accepted samples.
+- Keep the C implementation as a reference, but use the paper's units for `G`.
+
+## Stage 2: Local Verification
+
+- Unit-test packet encoding/decoding.
+- Unit-test estimator selection by minimum delay sum.
+- Unit-test bulk-length adaptation and pacing calculations.
+- Compile the standalone example.
+
+## Stage 3: Lab Validation
+
+- Discover the current link-local addresses and interface names on the Mac mini
+  ring via SSH.
+- Build or run the PBProbe example on the relevant remotes.
+- Run `iperf3` over the same link-local scoped addresses as the baseline.
+- Compare PBProbe estimates against `iperf3` with a reasonable tolerance.
+- If estimates are outside tolerance, adjust only algorithm parameters or
+  implementation bugs, not the scoring target.
+
+## Current Notes
+
+- The repo license is Apache-2.0. The dropped PBProbe source has a permissive
+  MIT-like license header with notice retention and academic citation language.
+- The C code appears to implement the core estimator, but its `G` sleep units
+  look inconsistent with the paper. This implementation should follow the paper.

rust/babblerd/shortcuts.md

@@ -0,0 +1,334 @@
+# `babblerd` Shortcuts
+
+This file tracks architectural and implementation shortcuts that were taken
+deliberately during the refactors. They are acceptable for now, but they are
+not meant to be the final design.
+
+This is not a dump of every `TODO` comment in the crate. It is the curated list
+of shortcuts that should be revisited later.
+
+## Architecture / IPC
+
+- The public control socket still uses an ad-hoc line protocol instead of the
+  intended `zbus`/D-Bus-style IPC surface.
+  Files:
+  - `src/daemon.rs`
+  - `src/main.rs`
+  Follow-up:
+  - Replace `keepalive <ttl_ms>` / `get-state` string commands with a typed IPC
+    API.
+
+- The daemon core currently tracks a single global keepalive deadline, not
+  per-client leases.
+  Files:
+  - `src/daemon.rs`
+  - `src/main.rs`
+  Why this is a shortcut:
+  - It does not model multiple clients independently.
+  - It cannot distinguish which client is keeping the service alive.
+  - The current tree also includes a temporary internal self-client in
+    `main.rs` that periodically issues keepalive commands just to keep the
+    daemon/routing stack alive during bring-up.
+  Follow-up:
+  - Introduce real lease ownership/tracking in the daemon core.
+  - Remove the temporary internal keepalive client once a real frontend or test
+    harness is driving the daemon.
+
+- Raw Babel debug output currently only goes to tracing logs.
+  Files:
+  - `src/babel/runtime.rs`
+  - `src/daemon.rs`
+  Why this is a shortcut:
+  - There is no configurable or structured diagnostics stream anymore.
+  - That is fine for now, but eventually debugging should not require tailing
+    daemon logs.
+  Follow-up:
+  - Add configurable debug output or a separate structured diagnostics stream
+    once the real IPC surface exists.
+
+- The daemon core exposes state only through `get-state` polling and inline
+  command responses.
+  Files:
+  - `src/daemon.rs`
+  Follow-up:
+  - Add real state publication/signals once the IPC surface is upgraded.
+
+## Service Lifecycle
+
+- The daemon now has explicit `Off/Starting/On/Stopping`, but the control model
+  is still minimal.
+  Files:
+  - `src/daemon.rs`
+  Why this is a shortcut:
+  - There is no richer lifecycle API yet.
+  - There is no explicit enable/disable policy beyond keepalive-driven on/off.
+  Follow-up:
+  - Revisit the final lifecycle API once IPC is made real.
+
+- `ServiceState::On` currently means “the routing tasks were started”, not a
+  stronger readiness guarantee such as “babeld is healthy, has admitted
+  interfaces, and is actually usable for mesh forwarding”.
+  Files:
+  - `src/daemon.rs`
+  - `src/routing_stack.rs`
+  - `src/babel/runtime.rs`
+  Why this is a shortcut:
+  - The frontend may eventually want to distinguish process/task liveness from
+    actual routing readiness.
+  Follow-up:
+  - Add a separate readiness field or richer public state model instead of
+    overloading `ServiceState::On`.
+
+- The resident TUN vs heavy routing-stack split is now in place, but the
+  naming and abstractions are still transitional.
+  Files:
+  - `src/daemon.rs`
+  - `src/routing_stack.rs`
+  - `src/tun.rs`
+  Follow-up:
+  - Revisit names and boundaries after the daemon core / IPC architecture settles.
+
+- `RoutingStack::stop` still uses abort-driven shutdown for the interface
+  watcher and logger task.
+  Files:
+  - `src/routing_stack.rs`
+  Why this is a shortcut:
+  - It is pragmatic, but not a carefully coordinated shutdown protocol.
+  Follow-up:
+  - Replace task abortion with explicit shutdown signaling where it matters.
+
+## Babel Integration
+
+- `babeld` runtime startup config is still assembled partly as raw strings.
+  Files:
+  - `src/babel/runtime.rs`
+  - `src/babel/command.rs`
+  Why this is a shortcut:
+  - The local-socket command side is typed, but spawn-time `-C` config is not.
+  Follow-up:
+  - Add a typed Babel config/config-statement layer.
+
+- The runtime still depends on fork-specific `babeld` behavior
+  (`kernel-install false`) while spawning `"babeld"` from `PATH`.
+  Files:
+  - `src/babel/runtime.rs`
+  - `../nix/babeld.nix`
+  Why this is a shortcut:
+  - It assumes the right binary is on `PATH`.
+  - The Nix packaging is still not pinned to a specific revision.
+  Follow-up:
+  - Pin the fork revision and make the runtime use that exact binary.
+
+## Networking / Interface Admission
+
+- Interface admission is still heuristic and too broad on macOS.
+  Files:
+  - `src/lib.rs` (`if_watcher`)
+  - `src/config.rs`
+  - `src/fib.rs`
+  Why this is a shortcut:
+  - Any `en*` interface with link-local IPv6 and `is_up()` can still get pulled
+    into Babel during bootstrap.
+  - This can include unrelated Wi‑Fi, built-in Ethernet, USB Ethernet, etc.
+  - The dataplane now narrows that broad bootstrap set back down to interfaces
+    that actually have live Babel neighbours, which is much closer to the real
+    transport set.
+  - But the watcher/bootstrap side is still using the coarse `en*` heuristic,
+    and the env allowlist is still just a bring-up escape hatch rather than the
+    long-term admission policy.
+  - When multiple admissible wired links exist, Babel's current wired scoring
+    still treats them essentially flatly, so path selection is based on Babel's
+    existing costs rather than measured latency/throughput differences between
+    those direct links.
+  Follow-up:
+  - Replace the watcher-side bootstrap heuristic with a stronger admission
+    policy (neighbor proof, richer metadata, or both), so Babel does not need
+    broad speculative interface admission just to discover the right links.
+  - Add a future link-quality scoring path so broadly admissible direct links
+    can still be ranked by actual observed quality rather than flat wired cost.
+
+- The dataplane now derives immutable FIB snapshots and runs on a dedicated
+  thread, but it still assumes interface names are the stable long-lived
+  identity for socket ownership.
+  Files:
+  - `src/fib.rs`
+  - `src/dataplane.rs`
+  Why this is a shortcut:
+  - The dataplane now owns sockets from the admitted interface set rather than
+    inferring them only from current routes, and it refreshes retained sockets
+    when a name resolves to a new ifindex.
+  - The dataplane now also has a lightweight timer-driven reconcile retry for
+    admitted interfaces whose socket setup was skipped or failed, so unchanged
+    FIB snapshots no longer suppress retries completely.
+  - That fixes the earlier route-derived and stale-ifindex bugs, but the design
+    still assumes interface names are stable enough to be the long-lived
+    control-plane identity.
+  Follow-up:
+  - Revisit whether the long-term identity should be richer than `ifname`,
+    especially if interface renames/hotplug churn become common during runtime.
+  - On macOS in particular, the current "one socket per interface" receive
+    model is not trustworthy enough to identify the real ingress interface:
+    live testing shows packets sent directly over one Thunderbolt link can be
+    received on a different `MioUdpSocket` while the peer scope-id still
+    reflects the actual physical ingress interface.
+  - Revisit receive-side interface attribution on macOS, likely using ancillary
+    packet-info / receive-interface metadata instead of assuming the receiving
+    socket tells the truth.
+
+- The current dataplane is intentionally minimal and still drops several packet
+  classes silently.
+  Files:
+  - `src/dataplane.rs`
+  - `src/fib.rs`
+  - `src/routing_stack.rs`
+  Why this is a shortcut:
+  - The current lab state has reliable ICMPv6 reachability and now also basic
+    generic TCP correctness after convergence: a direct `nc` TCP send across
+    the overlay succeeds and the server receives the payload.
+  - But sustained throughput is still not healthy. In live `iperf3` testing, a
+    TCP flow transfers an initial burst and then stalls with heavy retransmits
+    and near-zero receive-side throughput.
+  - The current code also still treats `WouldBlock` on UDP send and TUN
+    reinjection as drop-on-backpressure behavior. That is now visible in logs,
+    but it is not yet a proper queued/backpressured forwarding model.
+  - So the remaining blocker is no longer "can non-ICMP traffic work at all",
+    but "why does sustained transport performance collapse under load".
+  - Restart-sensitive failures are now narrowed further than "the dataplane is
+    broken". In live `e11 -> e16` debugging, `e11` emitted the encapsulated
+    packet on the expected direct link, `e16` received it, delivered it into
+    the TUN, and the local stack generated a reply. The failure happened on the
+    return path because `e16`'s current FIB resolved the reply toward `en1`
+    instead of the direct link back to `e11`.
+  - That means the current post-restart blackholes are now dominated by
+    control-plane / FIB route selection under broad admissibility, not by the
+    dataplane failing to decapsulate or reinject packets.
+  - There is no ICMPv6 Time Exceeded generation yet.
+  - There is no Packet Too Big handling yet.
+  - No-route and invalid-packet cases are mostly tracing-and-drop behavior.
+  Follow-up:
+  - Characterize the throughput collapse under load, starting with route churn,
+    multi-interface path selection, and loss/retransmit behavior on the direct
+    lab edge.
+  - Investigate why the current Babel-selected installed routes can prefer
+    higher-cost `en1` return paths over direct Thunderbolt neighbours after
+    restart, and how that should interact with the eventual measured link
+    scoring policy.
+  - Add proper ICMPv6 error generation and tighter packet-validation behavior
+    once the first end-to-end forwarding path is validated.
+
+- `TunDevice` is still a thin platform-specific wrapper with some rough edges.
+  Files:
+  - `src/tun.rs`
+  - `src/dataplane.rs`
+  Why this is a shortcut:
+  - It still stores the address as `Ipv6Net` even though usage is `/128`-only.
+  - On macOS, the actual kernel interface name is still `utunN`; the daemon's
+    cross-platform naming has been cleaned up, but the OS-level interface name
+    is not under our control there.
+  - It still has hard-coded MTU and other tun-rs builder assumptions.
+  - The dataplane still relies on `mio::unix::SourceFd` and `AsRawFd` to poll
+    the TUN fd on Unix.
+  - On macOS, packet I/O must still go through `tun-rs`'s `SyncDevice::recv`
+    and `SyncDevice::send`; bypassing those with raw fd `read`/`write` breaks
+    utun packet-information handling even if `mio` polling itself is correct.
+  - That low-level fd borrowing is smaller and safer than the old
+    `unsafe`/owned-fd handoff, but it still keeps raw-fd details in the
+    dataplane hot path.
+  Follow-up:
+  - Tighten the type and revisit the platform-specific tuning once the dataplane
+    is implemented.
+  - Revisit whether a future dataplane/eventing design can remove the direct
+    `mio`/raw-fd dependency entirely.
+
+- The current MTU model is still intentionally crude:
+  - assume physical links must support 1500-byte packets,
+  - derive TUN MTU as `1500 - 40 (IPv6) - 8 (UDP) = 1452`,
+  - reject candidate physical interfaces below 1500 MTU.
+  Files:
+  - `src/config.rs`
+  - `src/lib.rs`
+  - `src/tun.rs`
+  Why this is a shortcut:
+  - It does not handle PMTUD, VLAN overhead, per-route MTU variation, or
+    jumbo-frame opportunities.
+  Follow-up:
+  - Replace the current fixed MTU model with route-aware MTU derivation once the
+    UDP dataplane exists.
+
+- The overlay route controller currently claims the whole overlay prefix
+  aggressively.
+  Files:
+  - `src/route_ctl.rs`
+  Why this is a shortcut:
+  - It removes any existing route matching `EXO_ULA_PREFIX` before adding the
+    daemon's own interface route, and removes all matching routes again on
+    shutdown.
+  - That is acceptable only if babblerd is the sole owner of the overlay
+    prefix.
+  Follow-up:
+  - Narrow route deletion so it only removes routes that this daemon installed,
+    or otherwise encode route ownership more precisely.
+
+## Identity / Security / Filesystem
+
+- The node-id file is created with `0600`, but existing files are only
+  owner-checked, not mode-checked.
+  Files:
+  - `src/identity.rs`
+  Why this is a shortcut:
+  - A root-owned but group/world-writable file would still be accepted.
+  Follow-up:
+  - Enforce safe permissions on reload, not just on initial creation.
+
+- The public IPC socket is intentionally world-accessible for now.
+  Files:
+  - `src/main.rs`
+  Why this is a shortcut:
+  - Any local user can connect, issue keepalives, and drive the daemon's public
+    control surface.
+  Follow-up:
+  - Revisit permissions/authz once the IPC surface is finalized.
+
+## Error Modeling
+
+- Several orchestration-layer errors are flattened to `String`/`Arc<str>` too
+  early.
+  Files:
+  - `src/daemon.rs`
+  - `src/babel/runtime.rs`
+  - `src/lib.rs` (`BabbleError::Other(String)`)
+  Why this is a shortcut:
+  - It loses structure and source-chain information.
+  Follow-up:
+  - Prefer typed errors or `eyre::Report` internally, and stringify only at the
+    IPC/UI boundary.
+
+## Constants / Magic Values
+
+- A few important constants are still effectively magic values:
+  - EXO ULA prefix details
+  - default router UDP port
+  - various timeout/sleep durations in the Babel runtime
+  - tun MTU
+  Files:
+  - `src/config.rs`
+  - `src/babel/runtime.rs`
+  - `src/tun.rs`
+  Follow-up:
+  - Either justify them clearly as real protocol/runtime constants or move them
+    into better configuration/abstraction layers.
+
+## Testing
+
+- The typed Babel parser/state layers are tested, but the newer daemon-core and
+  routing-stack lifecycle behavior is still lightly tested.
+  Files:
+  - `src/daemon.rs`
+  - `src/routing_stack.rs`
+  - `src/main.rs`
+  Follow-up:
+  - Add focused tests for:
+    - keepalive-driven transitions,
+    - stack start/stop behavior,
+    - public socket command behavior,
+    - failure propagation from the routing stack.

rust/babblerd/src/babel/command.rs

@@ -0,0 +1,71 @@
+//! Typed representation of commands sent to `babeld`'s local socket.
+//!
+//! This is the outbound counterpart to [`crate::babel::line`]:
+//!
+//! - [`crate::babel::line`] models what `babeld` emits
+//! - this module models the runtime control lines that `babblerd` sends
+//!
+//! The scope here is intentionally narrow: this module only models the local-socket
+//! commands that `babblerd` currently issues at runtime.
+//!
+//! NOTE: spawn-time `-C` configuration strings are still assembled in the runtime layer for now.
+//!       If you want to push the protocol model further, the next obvious extraction is a typed
+//!       configuration/config-statement layer rather than more runtime socket commands.
+
+use std::fmt;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum BabelCommand {
+    Dump,
+    Monitor,
+    Unmonitor,
+    Quit,
+    Interface(Box<str>),
+}
+
+impl BabelCommand {
+    /// Encode this command for the local `babeld` socket, including line framing.
+    #[must_use]
+    pub fn encode(&self) -> String {
+        format!("{self}\n")
+    }
+}
+
+impl fmt::Display for BabelCommand {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Dump => f.write_str("dump"),
+            Self::Monitor => f.write_str("monitor"),
+            Self::Unmonitor => f.write_str("unmonitor"),
+            Self::Quit => f.write_str("quit"),
+            Self::Interface(ifname) => write!(f, "interface {ifname}"),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::BabelCommand;
+
+    #[test]
+    fn renders_commands() {
+        assert_eq!(BabelCommand::Dump.to_string(), "dump");
+        assert_eq!(BabelCommand::Monitor.to_string(), "monitor");
+        assert_eq!(BabelCommand::Unmonitor.to_string(), "unmonitor");
+        assert_eq!(BabelCommand::Quit.to_string(), "quit");
+        assert_eq!(
+            BabelCommand::Interface("en2".into()).to_string(),
+            "interface en2"
+        );
+    }
+
+    #[test]
+    fn encodes_commands() {
+        assert_eq!(BabelCommand::Dump.encode(), "dump\n");
+        assert_eq!(BabelCommand::Monitor.encode(), "monitor\n");
+        assert_eq!(
+            BabelCommand::Interface("en2".into()).encode(),
+            "interface en2\n"
+        );
+    }
+}

rust/babblerd/src/babel/line.rs

@@ -0,0 +1,686 @@
+//! Typed representation of lines emitted by `babeld`'s local socket.
+//!
+//! This module models the inbound side of the Babel local control protocol:
+//!
+//! - [`BabelLine`] is one parsed wire line.
+//! - [`HeaderLine`] covers the connection prelude.
+//! - [`Status`] covers command completion lines such as `ok`, `bad`, and `no ...`.
+//! - [`Event`] and its associated structs cover the asynchronous routing/interface updates
+//!   emitted by `dump` and `monitor`.
+//!
+//! The sibling parser lives in [`parse`]. Its job is to turn raw socket lines into these domain
+//! types. Higher layers such as the Babel runtime/state code should depend on this module's
+//! types, and keep raw strings only at the actual socket boundary.
+//!
+//! More concretely:
+//!
+//! - use [`parse::parse_line`] when reading from `babeld`
+//! - reduce [`Event`] values into [`crate::babel::state::BabelState`]
+//! - treat [`Status`] as command acknowledgements
+//! - keep outbound socket/config commands in a separate module rather than mixing them into
+//!   this inbound line model
+
+use crate::babel::Eui64;
+use ipnet::IpNet;
+use std::net::{IpAddr, Ipv4Addr};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum BabelLine {
+    Header(HeaderLine),
+    Status(Status),
+    Event(Event),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum HeaderLine {
+    Banner { major: u8, minor: u8 },
+    Version(Box<str>),
+    Host(Box<str>),
+    MyId(Eui64),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Status {
+    Ok,
+    Bad,
+    No(Option<Box<str>>),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Event {
+    Interface(InterfaceEvent),
+    Neighbour(NeighbourEvent),
+    XRoute(XRouteEvent),
+    Route(RouteEvent),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EventKind {
+    Add,
+    Change,
+    Flush,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct InterfaceEvent {
+    pub kind: EventKind,
+    pub ifname: Box<str>,
+    pub up: bool,
+    pub ipv6: Option<IpAddr>,
+    pub ipv4: Option<Ipv4Addr>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct NeighbourEvent {
+    pub kind: EventKind,
+    pub handle: u64,
+    pub address: IpAddr,
+    pub ifname: Box<str>,
+    pub reach: u16,
+    pub ureach: u16,
+    pub rxcost: u32,
+    pub txcost: u32,
+    pub rtt_millis: Option<u32>,
+    pub rttcost: Option<u32>,
+    pub cost: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct XRouteEvent {
+    pub kind: EventKind,
+    pub prefix: IpNet,
+    pub from: IpNet,
+    pub metric: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RouteEvent {
+    pub kind: EventKind,
+    pub handle: u64,
+    pub prefix: IpNet,
+    pub from: IpNet,
+    pub installed: bool,
+    pub id: Eui64,
+    pub metric: u32,
+    pub refmetric: u32,
+    pub via: IpAddr,
+    pub ifname: Box<str>,
+}
+
+/// Parser for `babeld`'s local socket output.
+///
+/// This submodule is the wire-format counterpart to the parent [`crate::babel::line`] domain
+/// types. It turns raw socket text into [`BabelLine`] values.
+///
+/// The local socket protocol implemented in `networking-related/babeld/local.c` is line-oriented
+/// ASCII. The parser is split into two layers:
+///
+/// - [`RawLines`] does zero-copy line framing over buffered bytes with [`memchr`].
+/// - [`parse_line`] parses one complete line with [`winnow`].
+/// - [`ParsedLines`] is a convenience adapter for buffered transcripts such as `dump` output.
+///
+/// `monitor` mode uses the exact same line grammar as `dump`; it simply keeps emitting event lines
+/// after the initial snapshot.
+///
+/// The accepted grammar is:
+///
+/// ```text
+/// stream     ::= (line "\n")* line?
+/// line       ::= header | status | event
+///
+/// header     ::= banner | version | host | my-id
+/// banner     ::= "BABEL " uint "." uint
+/// version    ::= "version " text
+/// host       ::= "host " text
+/// my-id      ::= "my-id " eui64
+///
+/// status     ::= "ok" | "bad" | ("no" (" " text)?)
+///
+/// event      ::= kind " " (interface | neighbour | xroute | route)
+/// kind       ::= "add" | "change" | "flush"
+///
+/// interface  ::= "interface " ifname " up " bool
+///               (" ipv6 " ip)?
+///               (" ipv4 " ipv4)?
+///
+/// neighbour  ::= "neighbour " hex " address " ip " if " ifname
+///               " reach " hex " ureach " hex
+///               " rxcost " uint " txcost " uint
+///               (" rtt " millis " rttcost " uint)?
+///               " cost " uint
+///
+/// xroute     ::= "xroute " prefix "-" prefix
+///               " prefix " prefix " from " prefix " metric " uint
+///
+/// route      ::= "route " hex
+///               " prefix " prefix " from " prefix
+///               " installed " yesno
+///               " id " eui64
+///               " metric " uint " refmetric " uint
+///               " via " ip " if " ifname
+/// ```
+///
+/// The accepted grammar is written in a regex/BNF-ish notation:
+///
+/// - `e1 e2` means concatenation
+/// - `e1 | e2` means choice
+/// - `e*` means zero or more
+/// - `e+` means one or more
+/// - `e?` means optional
+/// - `(e)` groups expressions
+///
+/// # Notes
+///
+/// - The `xroute` summary `prefix-from` token is parsed only to consume the wire format;
+///   the later `prefix` and `from` fields are treated as the authoritative values.
+/// - The parser is intentionally strict about the documented token set. Internal defensive
+///   fallbacks in `babeld` such as `???` are not treated as part of the formal grammar.
+pub mod parse {
+    use crate::babel::Eui64;
+    use crate::babel::line::{
+        BabelLine, Event, EventKind, HeaderLine, InterfaceEvent, NeighbourEvent, RouteEvent,
+        Status, XRouteEvent,
+    };
+    use ipnet::IpNet;
+    use memchr::memchr;
+    use std::{
+        net::{IpAddr, Ipv4Addr},
+        str::FromStr,
+    };
+    use thiserror::Error;
+    use winnow::{
+        ascii::{dec_uint, hex_uint, space1},
+        combinator::{alt, eof, opt, preceded, terminated},
+        error::ContextError,
+        prelude::*,
+        token::{rest, take_till},
+    };
+
+    #[derive(Error, Debug)]
+    pub enum ParseError {
+        #[error("invalid utf8 in babeld output: {0}")]
+        InvalidUtf8(#[from] std::str::Utf8Error),
+        #[error("failed to parse babeld line {line:?}: {error}")]
+        Syntax { line: String, error: String },
+    }
+
+    /// Zero-copy line framing for already-buffered socket output.
+    ///
+    /// This is the `stream = { line }` part of the grammar: framing happens first,
+    /// then each line is parsed independently by `parse_line`.
+    #[derive(Debug, Clone)]
+    pub struct RawLines<'a> {
+        remaining: &'a [u8],
+    }
+
+    impl<'a> RawLines<'a> {
+        pub fn new(bytes: &'a [u8]) -> Self {
+            Self { remaining: bytes }
+        }
+    }
+
+    impl<'a> Iterator for RawLines<'a> {
+        type Item = Result<&'a str, ParseError>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+            if self.remaining.is_empty() {
+                return None;
+            }
+
+            let split = memchr(b'\n', self.remaining);
+            let (line, rest_bytes) = match split {
+                Some(idx) => (&self.remaining[..idx], &self.remaining[idx + 1..]),
+                None => (self.remaining, &[][..]),
+            };
+            self.remaining = rest_bytes;
+
+            let line = if let Some(stripped) = line.strip_suffix(b"\r") {
+                stripped
+            } else {
+                line
+            };
+
+            Some(std::str::from_utf8(line).map_err(ParseError::InvalidUtf8))
+        }
+    }
+
+    /// Convenience adapter for parsing a fully buffered transcript, e.g. a dump.
+    #[derive(Debug, Clone)]
+    pub struct ParsedLines<'a> {
+        raw: RawLines<'a>,
+    }
+
+    impl<'a> ParsedLines<'a> {
+        pub fn new(bytes: &'a [u8]) -> Self {
+            Self {
+                raw: RawLines::new(bytes),
+            }
+        }
+    }
+
+    impl<'a> Iterator for ParsedLines<'a> {
+        type Item = Result<BabelLine, ParseError>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+            self.raw.next().map(|line| line.and_then(parse_line))
+        }
+    }
+
+    pub fn parse_line(line: &str) -> Result<BabelLine, ParseError> {
+        terminated(parse_babel_line, eof)
+            .parse(line)
+            .map_err(|err| ParseError::Syntax {
+                line: line.to_owned(),
+                error: err.to_string(),
+            })
+    }
+
+    fn parse_babel_line(input: &mut &str) -> ModalResult<BabelLine> {
+        alt((
+            parse_banner,
+            parse_version,
+            parse_host,
+            parse_my_id,
+            parse_ok,
+            parse_bad,
+            parse_no,
+            parse_event,
+        ))
+        .parse_next(input)
+    }
+
+    fn parse_banner(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "BABEL ".parse_next(input)?;
+        let major = dec_uint::<_, u8, _>.parse_next(input)?;
+        let _ = '.'.parse_next(input)?;
+        let minor = dec_uint::<_, u8, _>.parse_next(input)?;
+        Ok(BabelLine::Header(HeaderLine::Banner { major, minor }))
+    }
+
+    fn parse_version(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "version ".parse_next(input)?;
+        let version = Box::<str>::from(rest.parse_next(input)?);
+        Ok(BabelLine::Header(HeaderLine::Version(version)))
+    }
+
+    fn parse_host(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "host ".parse_next(input)?;
+        let host = Box::<str>::from(rest.parse_next(input)?);
+        Ok(BabelLine::Header(HeaderLine::Host(host)))
+    }
+
+    fn parse_my_id(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "my-id ".parse_next(input)?;
+        let id = parse_eui64.parse_next(input)?;
+        Ok(BabelLine::Header(HeaderLine::MyId(id)))
+    }
+
+    fn parse_ok(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "ok".parse_next(input)?;
+        Ok(BabelLine::Status(Status::Ok))
+    }
+
+    fn parse_bad(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "bad".parse_next(input)?;
+        Ok(BabelLine::Status(Status::Bad))
+    }
+
+    fn parse_no(input: &mut &str) -> ModalResult<BabelLine> {
+        let _ = "no".parse_next(input)?;
+        let message = opt(preceded(space1, rest)).parse_next(input)?;
+        let message = message.filter(|msg| !msg.is_empty()).map(Into::into);
+        Ok(BabelLine::Status(Status::No(message)))
+    }
+
+    fn parse_event(input: &mut &str) -> ModalResult<BabelLine> {
+        let kind = parse_kind.parse_next(input)?;
+        let _ = ' '.parse_next(input)?;
+        let entity = parse_word.parse_next(input)?;
+
+        match entity {
+            "interface" => parse_interface_event(kind, input).map(Event::Interface),
+            "neighbour" => parse_neighbour_event(kind, input).map(Event::Neighbour),
+            "xroute" => parse_xroute_event(kind, input).map(Event::XRoute),
+            "route" => parse_route_event(kind, input).map(Event::Route),
+            _ => Err(winnow::error::ErrMode::Backtrack(ContextError::new())),
+        }
+        .map(BabelLine::Event)
+    }
+
+    fn parse_interface_event(kind: EventKind, input: &mut &str) -> ModalResult<InterfaceEvent> {
+        let _ = ' '.parse_next(input)?;
+        let ifname = parse_word.parse_next(input)?;
+        let _ = " up ".parse_next(input)?;
+        let up = parse_bool.parse_next(input)?;
+        let ipv6 = opt(preceded(" ipv6 ", parse_ip_addr)).parse_next(input)?;
+        let ipv4 = opt(preceded(" ipv4 ", parse_ipv4_addr)).parse_next(input)?;
+
+        Ok(InterfaceEvent {
+            kind,
+            ifname: ifname.into(),
+            up,
+            ipv6,
+            ipv4,
+        })
+    }
+
+    fn parse_neighbour_event(kind: EventKind, input: &mut &str) -> ModalResult<NeighbourEvent> {
+        let _ = ' '.parse_next(input)?;
+        let handle = parse_hex_u64.parse_next(input)?;
+        let _ = " address ".parse_next(input)?;
+        let address = parse_ip_addr.parse_next(input)?;
+        let _ = " if ".parse_next(input)?;
+        let ifname = parse_word.parse_next(input)?;
+        let _ = " reach ".parse_next(input)?;
+        let reach = parse_hex_u16.parse_next(input)?;
+        let _ = " ureach ".parse_next(input)?;
+        let ureach = parse_hex_u16.parse_next(input)?;
+        let _ = " rxcost ".parse_next(input)?;
+        let rxcost = dec_uint::<_, u32, _>.parse_next(input)?;
+        let _ = " txcost ".parse_next(input)?;
+        let txcost = dec_uint::<_, u32, _>.parse_next(input)?;
+        let rtt = opt(parse_rtt_clause).parse_next(input)?;
+        let _ = " cost ".parse_next(input)?;
+        let cost = dec_uint::<_, u32, _>.parse_next(input)?;
+
+        Ok(NeighbourEvent {
+            kind,
+            handle,
+            address,
+            ifname: ifname.into(),
+            reach,
+            ureach,
+            rxcost,
+            txcost,
+            rtt_millis: rtt.map(|(millis, _)| millis),
+            rttcost: rtt.map(|(_, cost)| cost),
+            cost,
+        })
+    }
+
+    fn parse_xroute_event(kind: EventKind, input: &mut &str) -> ModalResult<XRouteEvent> {
+        let _ = ' '.parse_next(input)?;
+        let _summary_prefix = parse_prefix_until('-').parse_next(input)?;
+        let _ = '-'.parse_next(input)?;
+        let _summary_from = parse_prefix.parse_next(input)?;
+        let _ = " prefix ".parse_next(input)?;
+        let prefix = parse_prefix.parse_next(input)?;
+        let _ = " from ".parse_next(input)?;
+        let from = parse_prefix.parse_next(input)?;
+        let _ = " metric ".parse_next(input)?;
+        let metric = dec_uint::<_, u32, _>.parse_next(input)?;
+
+        Ok(XRouteEvent {
+            kind,
+            prefix,
+            from,
+            metric,
+        })
+    }
+
+    fn parse_route_event<'a>(kind: EventKind, input: &mut &'a str) -> ModalResult<RouteEvent> {
+        let _ = ' '.parse_next(input)?;
+        let handle = parse_hex_u64.parse_next(input)?;
+        let _ = " prefix ".parse_next(input)?;
+        let prefix = parse_prefix.parse_next(input)?;
+        let _ = " from ".parse_next(input)?;
+        let from = parse_prefix.parse_next(input)?;
+        let _ = " installed ".parse_next(input)?;
+        let installed = parse_yes_no.parse_next(input)?;
+        let _ = " id ".parse_next(input)?;
+        let id = parse_eui64.parse_next(input)?;
+        let _ = " metric ".parse_next(input)?;
+        let metric = dec_uint::<_, u32, _>.parse_next(input)?;
+        let _ = " refmetric ".parse_next(input)?;
+        let refmetric = dec_uint::<_, u32, _>.parse_next(input)?;
+        let _ = " via ".parse_next(input)?;
+        let via = parse_ip_addr.parse_next(input)?;
+        let _ = " if ".parse_next(input)?;
+        let ifname = parse_word.parse_next(input)?;
+
+        Ok(RouteEvent {
+            kind,
+            handle,
+            prefix,
+            from,
+            installed,
+            id,
+            metric,
+            refmetric,
+            via,
+            ifname: ifname.into(),
+        })
+    }
+
+    fn parse_rtt_clause(input: &mut &str) -> ModalResult<(u32, u32)> {
+        let _ = " rtt ".parse_next(input)?;
+        let millis = parse_millis.parse_next(input)?;
+        let _ = " rttcost ".parse_next(input)?;
+        let rttcost = dec_uint::<_, u32, _>.parse_next(input)?;
+        Ok((millis, rttcost))
+    }
+
+    fn parse_kind(input: &mut &str) -> ModalResult<EventKind> {
+        alt((
+            "add".value(EventKind::Add),
+            "change".value(EventKind::Change),
+            "flush".value(EventKind::Flush),
+        ))
+        .parse_next(input)
+    }
+
+    fn parse_bool(input: &mut &str) -> ModalResult<bool> {
+        alt(("true".value(true), "false".value(false))).parse_next(input)
+    }
+
+    fn parse_yes_no(input: &mut &str) -> ModalResult<bool> {
+        alt(("yes".value(true), "no".value(false))).parse_next(input)
+    }
+
+    fn parse_ip_addr(input: &mut &str) -> ModalResult<IpAddr> {
+        parse_word.try_map(IpAddr::from_str).parse_next(input)
+    }
+
+    fn parse_ipv4_addr(input: &mut &str) -> ModalResult<Ipv4Addr> {
+        parse_word.try_map(Ipv4Addr::from_str).parse_next(input)
+    }
+
+    fn parse_prefix(input: &mut &str) -> ModalResult<IpNet> {
+        parse_word.try_map(IpNet::from_str).parse_next(input)
+    }
+
+    fn parse_prefix_until(separator: char) -> impl FnMut(&mut &str) -> ModalResult<IpNet> {
+        move |input: &mut &str| {
+            let token = take_till(1.., |c: char| c == separator).parse_next(input)?;
+            IpNet::from_str(token)
+                .map_err(|_| winnow::error::ErrMode::Backtrack(ContextError::new()))
+        }
+    }
+    fn parse_eui64(input: &mut &str) -> ModalResult<Eui64> {
+        parse_word.try_map(Eui64::from_str).parse_next(input)
+    }
+
+    fn parse_hex_u64(input: &mut &str) -> ModalResult<u64> {
+        hex_uint.parse_next(input)
+    }
+
+    fn parse_hex_u16(input: &mut &str) -> ModalResult<u16> {
+        hex_uint.parse_next(input)
+    }
+
+    fn parse_millis(input: &mut &str) -> ModalResult<u32> {
+        let word = parse_word.parse_next(input)?;
+        parse_millis_str(word).map_err(|_| winnow::error::ErrMode::Backtrack(ContextError::new()))
+    }
+
+    fn parse_word<'a>(input: &mut &'a str) -> ModalResult<&'a str> {
+        take_till(1.., |c: char| c == ' ').parse_next(input)
+    }
+
+    fn parse_millis_str(value: &str) -> Result<u32, &'static str> {
+        let (secs, millis) = value
+            .split_once('.')
+            .ok_or("missing milliseconds separator")?;
+        if millis.len() != 3 || !millis.bytes().all(|b| b.is_ascii_digit()) {
+            return Err("expected 3-digit millisecond suffix");
+        }
+        let secs = secs
+            .parse::<u32>()
+            .map_err(|_| "invalid seconds field in rtt value")?;
+        let millis = millis
+            .parse::<u32>()
+            .map_err(|_| "invalid milliseconds field in rtt value")?;
+        secs.checked_mul(1000)
+            .and_then(|s| s.checked_add(millis))
+            .ok_or("rtt value overflowed u32")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::babel::line::parse::{ParsedLines, parse_line};
+    use std::str::FromStr;
+
+    #[test]
+    fn parse_header_banner() {
+        assert_eq!(
+            parse_line("BABEL 1.0").unwrap(),
+            BabelLine::Header(HeaderLine::Banner { major: 1, minor: 0 })
+        );
+    }
+
+    #[test]
+    fn parse_header_metadata() {
+        assert_eq!(
+            parse_line("version babeld-1.13.1").unwrap(),
+            BabelLine::Header(HeaderLine::Version("babeld-1.13.1".into()))
+        );
+        assert_eq!(
+            parse_line("host e2").unwrap(),
+            BabelLine::Header(HeaderLine::Host("e2".into()))
+        );
+        assert_eq!(
+            parse_line("my-id 02:00:00:00:00:00:00:01").unwrap(),
+            BabelLine::Header(HeaderLine::MyId(Eui64::new(2, 0, 0, 0, 0, 0, 0, 1)))
+        );
+    }
+
+    #[test]
+    fn parse_status_lines() {
+        assert_eq!(parse_line("ok").unwrap(), BabelLine::Status(Status::Ok));
+        assert_eq!(parse_line("bad").unwrap(), BabelLine::Status(Status::Bad));
+        assert_eq!(
+            parse_line("no No such interface").unwrap(),
+            BabelLine::Status(Status::No(Some("No such interface".into())))
+        );
+    }
+
+    #[test]
+    fn parse_interface_event() {
+        assert_eq!(
+            parse_line("add interface en2 up true ipv6 fe80::1 ipv4 169.254.1.2").unwrap(),
+            BabelLine::Event(Event::Interface(InterfaceEvent {
+                kind: EventKind::Add,
+                ifname: "en2".into(),
+                up: true,
+                ipv6: Some(IpAddr::from_str("fe80::1").unwrap()),
+                ipv4: Some(Ipv4Addr::new(169, 254, 1, 2)),
+            }))
+        );
+        assert_eq!(
+            parse_line("change interface en3 up false").unwrap(),
+            BabelLine::Event(Event::Interface(InterfaceEvent {
+                kind: EventKind::Change,
+                ifname: "en3".into(),
+                up: false,
+                ipv6: None,
+                ipv4: None,
+            }))
+        );
+    }
+
+    #[test]
+    fn parse_neighbour_event() {
+        assert_eq!(
+            parse_line(
+                "add neighbour 7ffdeadbeef address fe80::1 if en2 reach 00ff ureach 000f rxcost 256 txcost 96 rtt 0.123 rttcost 32 cost 128"
+            )
+                .unwrap(),
+            BabelLine::Event(Event::Neighbour(NeighbourEvent {
+                kind: EventKind::Add,
+                handle: 0x7ffdeadbeef,
+                address: IpAddr::from_str("fe80::1").unwrap(),
+                ifname: "en2".into(),
+                reach: 0x00ff,
+                ureach: 0x000f,
+                rxcost: 256,
+                txcost: 96,
+                rtt_millis: Some(123),
+                rttcost: Some(32),
+                cost: 128,
+            }))
+        );
+    }
+
+    #[test]
+    fn parse_xroute_event() {
+        assert_eq!(
+            parse_line(
+                "add xroute fd00::1/128-fd00::/64 prefix fd00::1/128 from fd00::/64 metric 0"
+            )
+            .unwrap(),
+            BabelLine::Event(Event::XRoute(XRouteEvent {
+                kind: EventKind::Add,
+                prefix: IpNet::from_str("fd00::1/128").unwrap(),
+                from: IpNet::from_str("fd00::/64").unwrap(),
+                metric: 0,
+            }))
+        );
+    }
+
+    #[test]
+    fn parse_route_event() {
+        assert_eq!(
+            parse_line(
+                "change route 7ffdeadbeef prefix fd00::1/128 from fd00::/64 installed yes id 02:00:00:00:00:00:00:01 metric 96 refmetric 0 via fe80::2 if en2"
+            )
+                .unwrap(),
+            BabelLine::Event(Event::Route(RouteEvent {
+                kind: EventKind::Change,
+                handle: 0x7ffdeadbeef,
+                prefix: IpNet::from_str("fd00::1/128").unwrap(),
+                from: IpNet::from_str("fd00::/64").unwrap(),
+                installed: true,
+                id: Eui64::new(2, 0, 0, 0, 0, 0, 0, 1),
+                metric: 96,
+                refmetric: 0,
+                via: IpAddr::from_str("fe80::2").unwrap(),
+                ifname: "en2".into(),
+            }))
+        );
+    }
+
+    #[test]
+    fn raw_lines_uses_memchr_framing() {
+        let bytes = b"BABEL 1.0\nok\nadd interface en2 up false\n";
+        let parsed = ParsedLines::new(bytes)
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+
+        assert_eq!(
+            parsed,
+            vec![
+                BabelLine::Header(HeaderLine::Banner { major: 1, minor: 0 }),
+                BabelLine::Status(Status::Ok),
+                BabelLine::Event(Event::Interface(InterfaceEvent {
+                    kind: EventKind::Add,
+                    ifname: "en2".into(),
+                    up: false,
+                    ipv6: None,
+                    ipv4: None,
+                })),
+            ]
+        );
+    }
+}

rust/babblerd/src/babel/mod.rs

@@ -0,0 +1,38 @@
+use ipnet::Ipv6Net;
+use std::sync::Arc;
+use tokio::sync::{mpsc, watch};
+
+use crate::Result;
+
+pub mod command;
+pub mod line;
+pub mod runtime;
+pub mod state;
+
+use runtime::BabelRuntime;
+
+/// An EUI-64 type aliased to [`macaddr::MacAddr8`].
+pub type Eui64 = macaddr::MacAddr8;
+pub use state::BabelState;
+
+#[derive(Debug)]
+pub enum Babble {
+    AddIface(Box<str>),
+}
+
+#[tracing::instrument(skip(state_send, recv))]
+pub async fn babel(
+    advertised: Ipv6Net,
+    mut recv: mpsc::Receiver<Babble>,
+    state_send: watch::Sender<Arc<BabelState>>,
+) -> Result<()> {
+    // Cannot spawn babeld without at least one interface to monitor.
+    let Some(Babble::AddIface(iface)) = recv.recv().await else {
+        return Ok(());
+    };
+
+    let mut runtime = BabelRuntime::spawn(advertised, &iface, state_send).await?;
+    let res1 = runtime.run(recv).await;
+    let res2 = runtime.shutdown().await;
+    res1.and(res2)
+}

rust/babblerd/src/babel/runtime.rs

@@ -0,0 +1,436 @@
+//! Managed `babeld` runtime for `babblerd`.
+//!
+//! This module owns the full lifecycle of the private `babeld` instance:
+//!
+//! - spawn-time configuration of the child process
+//! - the private Unix socket path used for the local control connection
+//! - connecting to that socket and speaking the local Babel protocol
+//! - running the monitor-driven control loop
+//! - shutdown and cleanup of the child process and socket
+//!
+//! Unlike the old `process` / `session` split, this is intended to model the real runtime unit:
+//! a single managed `babeld` process together with its single local control session.
+
+use std::fs::Permissions;
+use std::io;
+use std::os::unix::fs::PermissionsExt;
+use std::sync::Arc;
+
+use ipnet::Ipv6Net;
+use nix::errno::Errno;
+use nix::sys::signal::{Signal, kill};
+use nix::unistd::Pid;
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines};
+use tokio::net::UnixStream;
+use tokio::net::unix::{OwnedReadHalf, OwnedWriteHalf};
+use tokio::process::{Child, Command};
+use tokio::sync::{mpsc, watch};
+use tokio::time::{Duration, MissedTickBehavior, timeout};
+
+use crate::babel::Babble;
+use crate::babel::command::BabelCommand;
+use crate::babel::line::parse::ParseError;
+use crate::babel::line::{self, BabelLine, HeaderLine, Status};
+use crate::babel::state::BabelState;
+use crate::{BabbleError, Result};
+
+#[cfg(target_os = "macos")]
+const PRIVATE_SOCK_PATH: &str = "/var/run/babbler/private/babeld.sock";
+#[cfg(target_os = "linux")]
+const PRIVATE_SOCK_PATH: &str = "/run/babbler/private/babeld.sock";
+#[cfg(target_os = "macos")]
+const PRIVATE_DIR: &str = "/var/run/babbler/private";
+#[cfg(target_os = "linux")]
+const PRIVATE_DIR: &str = "/run/babbler/private";
+
+const STARTUP_SOCKET_TIMEOUT: Duration = Duration::from_secs(10);
+const STARTUP_SOCKET_POLL_INTERVAL: Duration = Duration::from_millis(50);
+const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
+
+pub(crate) struct BabelRuntime {
+    proc: Child,
+    read: Lines<BufReader<OwnedReadHalf>>,
+    write: OwnedWriteHalf,
+    state_send: watch::Sender<Arc<BabelState>>,
+    state: BabelState,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum StartupStage {
+    Banner,
+    Version,
+    Host,
+    MyId,
+    Ready,
+}
+
+impl StartupStage {
+    fn advance(self, line: BabelLine) -> Result<Option<Self>> {
+        match (self, line) {
+            (Self::Banner, BabelLine::Header(HeaderLine::Banner { major: 1, minor: 0 })) => {
+                Ok(Some(Self::Version))
+            }
+            (Self::Version, BabelLine::Header(HeaderLine::Version(_))) => Ok(Some(Self::Host)),
+            (Self::Host, BabelLine::Header(HeaderLine::Host(_))) => Ok(Some(Self::MyId)),
+            (Self::MyId, BabelLine::Header(HeaderLine::MyId(_))) => Ok(Some(Self::Ready)),
+            (Self::Ready, BabelLine::Status(Status::Ok)) => Ok(None),
+            (stage, other) => Err(BabbleError::Other(format!(
+                "unexpected babeld startup line while waiting for {stage:?}: {other:?}"
+            ))),
+        }
+    }
+}
+
+impl Drop for BabelRuntime {
+    #[inline]
+    fn drop(&mut self) {
+        // Emergency SIGKILL to avoid leaking an unmanaged babeld subprocess.
+        match self.proc.try_wait() {
+            Ok(None) => {}
+            Ok(Some(sc)) => {
+                if !sc.success() {
+                    _ = self.proc.start_kill();
+                }
+            }
+            _ => {
+                _ = self.proc.start_kill();
+            }
+        }
+    }
+}
+
+impl BabelRuntime {
+    #[tracing::instrument(skip(state_send))]
+    pub(crate) async fn spawn(
+        advertised: Ipv6Net,
+        iface: &str,
+        state_send: watch::Sender<Arc<BabelState>>,
+    ) -> Result<Self> {
+        tokio::fs::create_dir_all(PRIVATE_DIR).await?;
+        // TODO: remove this magic constant (and magic constants in general)
+        tokio::fs::set_permissions(PRIVATE_DIR, Permissions::from_mode(0o0700)).await?;
+        tracing::info!("spawning babeld socket in {PRIVATE_SOCK_PATH}");
+
+        let mut proc = match Command::new("babeld")
+            .arg("-G")
+            .arg(PRIVATE_SOCK_PATH)
+            .arg("-I")
+            .arg(format!("{PRIVATE_DIR}/babeld.pid"))
+            .arg("-C")
+            .arg("kernel-install false")
+            .arg("-C")
+            .arg(format!("redistribute local ip {advertised}"))
+            .arg("-C")
+            .arg("redistribute local deny")
+            .arg(iface)
+            .spawn()
+        {
+            Ok(proc) => proc,
+            Err(e) => {
+                tracing::warn!(error=%e, "failed to spawn babeld");
+                return Err(e.into());
+            }
+        };
+
+        if let Err(err) = Self::wait_for_socket(&mut proc).await {
+            Self::abort_child(&mut proc).await;
+            return Err(err);
+        }
+
+        // TODO: magic undocumented number
+        if let Err(err) =
+            std::fs::set_permissions(PRIVATE_SOCK_PATH, Permissions::from_mode(0o0600))
+        {
+            Self::abort_child(&mut proc).await;
+            return Err(err.into());
+        }
+
+        let (reader, write) = match UnixStream::connect(PRIVATE_SOCK_PATH).await {
+            Ok(stream) => stream.into_split(),
+            Err(err) => {
+                Self::abort_child(&mut proc).await;
+                return Err(err.into());
+            }
+        };
+
+        let mut runtime = Self {
+            proc,
+            read: BufReader::new(reader).lines(),
+            write,
+            state_send,
+            state: BabelState::new(),
+        };
+
+        if let Err(err) = runtime.await_ready().await {
+            let _ = runtime.shutdown().await;
+            return Err(err);
+        }
+
+        Ok(runtime)
+    }
+
+    async fn wait_for_socket(proc: &mut Child) -> Result<()> {
+        timeout(STARTUP_SOCKET_TIMEOUT, async {
+            let mut poll = tokio::time::interval(STARTUP_SOCKET_POLL_INTERVAL);
+            poll.set_missed_tick_behavior(MissedTickBehavior::Delay);
+
+            loop {
+                poll.tick().await;
+                match tokio::fs::try_exists(PRIVATE_SOCK_PATH).await {
+                    Ok(true) => return Ok(()),
+                    Ok(false) => {}
+                    Err(err) => return Err(err.into()),
+                }
+                if let Some(status) = proc.try_wait()? {
+                    return Err(BabbleError::BabeldCrashed(status.code()));
+                }
+            }
+        })
+        .await
+        .unwrap_or_else(|_| {
+            Err(BabbleError::Other(format!(
+                "timed out after {}s waiting for babeld socket {PRIVATE_SOCK_PATH}",
+                STARTUP_SOCKET_TIMEOUT.as_secs()
+            )))
+        })
+    }
+
+    async fn abort_child(proc: &mut Child) {
+        let _ = proc.kill().await;
+    }
+
+    #[tracing::instrument(skip_all)]
+    async fn await_ready(&mut self) -> Result<()> {
+        let mut stage = StartupStage::Banner;
+        while let Some(line) = self.read.next_line().await? {
+            match self.observe_line(line)? {
+                Ok(parsed) => match stage.advance(parsed)? {
+                    Some(next) => stage = next,
+                    None => {
+                        tracing::info!("babeld ok");
+                        return Ok(());
+                    }
+                },
+                Err(err) => {
+                    return Err(BabbleError::Other(format!(
+                        "failed to parse babeld startup prelude: {err}"
+                    )));
+                }
+            }
+        }
+        Err(BabbleError::Other(
+            "babeld closed before completing startup prelude".into(),
+        ))
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn query(&mut self, cmd: &BabelCommand) -> io::Result<Option<Status>> {
+        self.write.write_all(cmd.encode().as_bytes()).await?;
+        loop {
+            let Some(line) = self.read.next_line().await? else {
+                tracing::warn!("babeld closed unexpectedly");
+                return Ok(None);
+            };
+            match self.observe_line(line)? {
+                Ok(parsed) => {
+                    let status = self.reduce_live_line(parsed)?;
+                    let Some(status) = status else {
+                        continue;
+                    };
+                    match &status {
+                        Status::Ok => {}
+                        Status::Bad => tracing::warn!("malformed message sent to babeld"),
+                        Status::No(rest) => tracing::warn!("message rejected: {rest:?}"),
+                    }
+                    return Ok(Some(status));
+                }
+                Err(err) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::InvalidData,
+                        format!("failed to parse babeld command output: {err}"),
+                    ));
+                }
+            }
+        }
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn observe_line(&self, line: String) -> io::Result<std::result::Result<BabelLine, ParseError>> {
+        tracing::info!("[babel] {:?}", line);
+
+        let observed = match line::parse::parse_line(&line) {
+            Ok(parsed) => {
+                tracing::info!("[parsed] {:?}", parsed);
+                Ok(parsed)
+            }
+            Err(err) => {
+                tracing::error!(error=%err, "failed to parse babeld line");
+                Err(err)
+            }
+        };
+
+        Ok(observed)
+    }
+
+    #[tracing::instrument(skip(self))]
+    async fn start_monitoring(&mut self) -> io::Result<Option<Status>> {
+        let mut snapshot = BabelState::new();
+        self.write
+            .write_all(BabelCommand::Monitor.encode().as_bytes())
+            .await?;
+        loop {
+            let Some(line) = self.read.next_line().await? else {
+                tracing::warn!("babeld closed unexpectedly");
+                return Ok(None);
+            };
+            match self.observe_line(line)? {
+                Ok(BabelLine::Event(event)) => {
+                    snapshot.apply(event);
+                }
+                Ok(BabelLine::Status(status)) => {
+                    match &status {
+                        Status::Ok => {
+                            self.state = snapshot;
+                            self.publish_state();
+                        }
+                        Status::Bad => tracing::warn!("malformed message sent to babeld"),
+                        Status::No(rest) => tracing::warn!("message rejected: {rest:?}"),
+                    }
+                    return Ok(Some(status));
+                }
+                Ok(BabelLine::Header(header)) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::InvalidData,
+                        format!("unexpected header line during monitor bootstrap: {header:?}"),
+                    ));
+                }
+                Err(err) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::InvalidData,
+                        format!("failed to parse babeld monitor bootstrap output: {err}"),
+                    ));
+                }
+            }
+        }
+    }
+
+    fn publish_state(&self) {
+        self.state_send.send_replace(Arc::new(self.state.clone()));
+    }
+
+    fn reduce_live_line(&mut self, line: BabelLine) -> io::Result<Option<Status>> {
+        match line {
+            BabelLine::Event(event) => {
+                self.state.apply(event);
+                self.publish_state();
+                Ok(None)
+            }
+            BabelLine::Status(status) => Ok(Some(status)),
+            BabelLine::Header(header) => Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!("unexpected header line after startup: {header:?}"),
+            )),
+        }
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub(crate) async fn run(&mut self, mut recv: mpsc::Receiver<Babble>) -> Result<()> {
+        match self.start_monitoring().await? {
+            Some(Status::Ok) => {}
+            Some(Status::Bad) => {
+                return Err(BabbleError::Other(
+                    "babeld rejected monitor command as malformed".into(),
+                ));
+            }
+            Some(Status::No(reason)) => {
+                return Err(BabbleError::Other(format!(
+                    "babeld rejected monitor command: {reason:?}"
+                )));
+            }
+            None => {
+                return Err(BabbleError::Other(
+                    "babeld control socket closed during monitor bootstrap".into(),
+                ));
+            }
+        }
+
+        loop {
+            tokio::select! {
+                babble = recv.recv() => {
+                    tracing::debug!("[babble] {:?}", babble);
+                    let Some(babble) = babble else {
+                        break;
+                    };
+                    match babble {
+                        Babble::AddIface(iface) => {
+                            let cmd = BabelCommand::Interface(iface);
+                            self.query(&cmd).await?;
+                        }
+                    }
+                },
+                line = self.read.next_line() => {
+                    let line = match line {
+                        Ok(Some(line)) => line,
+                        Ok(None) => {
+                            return Err(BabbleError::Other(
+                                "babeld control socket closed during live monitoring".into(),
+                            ));
+                        }
+                        Err(err) => {
+                            return Err(BabbleError::Other(format!(
+                                "failed to read babeld control socket during live monitoring: {err}"
+                            )));
+                        }
+                    };
+                    match self.observe_line(line)? {
+                        Ok(parsed) => {
+                            if let Some(status) = self.reduce_live_line(parsed)? {
+                                tracing::debug!(?status, "ignoring unsolicited status line from babeld");
+                            }
+                        }
+                        Err(err) => {
+                            return Err(io::Error::new(
+                                io::ErrorKind::InvalidData,
+                                format!("failed to parse babeld monitor output: {err}"),
+                            )
+                            .into());
+                        }
+                    }
+                },
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn shutdown(mut self) -> Result<()> {
+        let kill_res = if let Some(pid) = self.proc.id() {
+            let pid: i32 = pid.try_into().expect("pid overflow");
+            let rc_err = match kill(Pid::from_raw(pid), Signal::SIGINT) {
+                Ok(()) | Err(Errno::ESRCH) => Ok(()),
+                Err(err) => Err(io::Error::from_raw_os_error(err as i32).into()),
+            };
+            match timeout(SHUTDOWN_TIMEOUT, self.proc.wait()).await {
+                Ok(Ok(code)) => {
+                    if code.success() {
+                        rc_err
+                    } else {
+                        rc_err.and_then(|()| Err(BabbleError::BabeldCrashed(code.code())))
+                    }
+                }
+                Ok(Err(e)) => Err(e.into()),
+                Err(_) => {
+                    self.proc.kill().await?;
+                    rc_err.and(Err(BabbleError::BabeldCrashed(None)))
+                }
+            }
+        } else {
+            Ok(())
+        };
+        let rem_res = match std::fs::remove_file(PRIVATE_SOCK_PATH) {
+            Ok(()) => Ok(()),
+            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
+            Err(e) => Err(e.into()),
+        };
+        kill_res.and(rem_res)
+    }
+}

rust/babblerd/src/babel/state.rs

@@ -0,0 +1,398 @@
+//! Reduced in-memory state derived from `babeld` event lines.
+//!
+//! This module is the consumer-side counterpart to [`crate::babel::line`]:
+//!
+//! - [`Event`] is the wire/domain event stream emitted by `babeld`
+//! - [`BabelState`] is the current snapshot obtained by reducing those events
+//!
+//! The reducer model is intentionally simple:
+//!
+//! - `add` inserts the entity into the relevant table
+//! - `change` upserts the entity into the relevant table
+//! - `flush` removes the entity from the relevant table
+//!
+//! The stored state types do **not** retain [`EventKind`], because the event
+//! kind is transport/update metadata rather than persistent object state.
+
+use std::collections::HashMap;
+use std::net::{IpAddr, Ipv4Addr};
+
+use crate::babel::Eui64;
+use crate::babel::line::{
+    Event, EventKind, InterfaceEvent, NeighbourEvent, RouteEvent, XRouteEvent,
+};
+use ipnet::IpNet;
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct BabelState {
+    pub interfaces: HashMap<Box<str>, InterfaceState>,
+    pub neighbours: HashMap<u64, NeighbourState>,
+    pub xroutes: HashMap<XRouteKey, XRouteState>,
+    pub routes: HashMap<u64, RouteState>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct InterfaceState {
+    pub ifname: Box<str>,
+    pub up: bool,
+    pub ipv6: Option<IpAddr>,
+    pub ipv4: Option<Ipv4Addr>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct NeighbourState {
+    pub handle: u64,
+    pub address: IpAddr,
+    pub ifname: Box<str>,
+    pub reach: u16,
+    pub ureach: u16,
+    pub rxcost: u32,
+    pub txcost: u32,
+    pub rtt_millis: Option<u32>,
+    pub rttcost: Option<u32>,
+    pub cost: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct XRouteKey {
+    pub prefix: IpNet,
+    pub from: IpNet,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct XRouteState {
+    pub prefix: IpNet,
+    pub from: IpNet,
+    pub metric: u32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RouteState {
+    pub handle: u64,
+    pub prefix: IpNet,
+    pub from: IpNet,
+    pub installed: bool,
+    pub id: Eui64,
+    pub metric: u32,
+    pub refmetric: u32,
+    pub via: IpAddr,
+    pub ifname: Box<str>,
+}
+
+impl BabelState {
+    #[must_use]
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn apply(&mut self, event: Event) {
+        match event {
+            Event::Interface(event) => self.apply_interface(event),
+            Event::Neighbour(event) => self.apply_neighbour(event),
+            Event::XRoute(event) => self.apply_xroute(event),
+            Event::Route(event) => self.apply_route(event),
+        }
+    }
+
+    pub fn extend<I>(&mut self, events: I)
+    where
+        I: IntoIterator<Item = Event>,
+    {
+        for event in events {
+            self.apply(event);
+        }
+    }
+
+    fn apply_interface(&mut self, event: InterfaceEvent) {
+        let key = event.ifname.clone();
+        match event.kind {
+            EventKind::Add | EventKind::Change => {
+                self.interfaces.insert(key, event.into());
+            }
+            EventKind::Flush => {
+                self.interfaces.remove(&key);
+            }
+        }
+    }
+
+    fn apply_neighbour(&mut self, event: NeighbourEvent) {
+        let key = event.handle;
+        match event.kind {
+            EventKind::Add | EventKind::Change => {
+                self.neighbours.insert(key, event.into());
+            }
+            EventKind::Flush => {
+                self.neighbours.remove(&key);
+            }
+        }
+    }
+
+    fn apply_xroute(&mut self, event: XRouteEvent) {
+        let key = XRouteKey {
+            prefix: event.prefix,
+            from: event.from,
+        };
+        match event.kind {
+            EventKind::Add | EventKind::Change => {
+                self.xroutes.insert(key, event.into());
+            }
+            EventKind::Flush => {
+                self.xroutes.remove(&key);
+            }
+        }
+    }
+
+    fn apply_route(&mut self, event: RouteEvent) {
+        let key = event.handle;
+        match event.kind {
+            EventKind::Add | EventKind::Change => {
+                self.routes.insert(key, event.into());
+            }
+            EventKind::Flush => {
+                self.routes.remove(&key);
+            }
+        }
+    }
+}
+
+impl From<InterfaceEvent> for InterfaceState {
+    fn from(event: InterfaceEvent) -> Self {
+        Self {
+            ifname: event.ifname,
+            up: event.up,
+            ipv6: event.ipv6,
+            ipv4: event.ipv4,
+        }
+    }
+}
+
+impl From<NeighbourEvent> for NeighbourState {
+    fn from(event: NeighbourEvent) -> Self {
+        Self {
+            handle: event.handle,
+            address: event.address,
+            ifname: event.ifname,
+            reach: event.reach,
+            ureach: event.ureach,
+            rxcost: event.rxcost,
+            txcost: event.txcost,
+            rtt_millis: event.rtt_millis,
+            rttcost: event.rttcost,
+            cost: event.cost,
+        }
+    }
+}
+
+impl From<XRouteEvent> for XRouteState {
+    fn from(event: XRouteEvent) -> Self {
+        Self {
+            prefix: event.prefix,
+            from: event.from,
+            metric: event.metric,
+        }
+    }
+}
+
+impl From<RouteEvent> for RouteState {
+    fn from(event: RouteEvent) -> Self {
+        Self {
+            handle: event.handle,
+            prefix: event.prefix,
+            from: event.from,
+            installed: event.installed,
+            id: event.id,
+            metric: event.metric,
+            refmetric: event.refmetric,
+            via: event.via,
+            ifname: event.ifname,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+
+    use super::{BabelState, InterfaceState, XRouteKey};
+    use crate::babel::Eui64;
+    use crate::babel::line::{
+        Event, EventKind, InterfaceEvent, NeighbourEvent, RouteEvent, XRouteEvent,
+    };
+    use ipnet::IpNet;
+
+    fn net(s: &str) -> IpNet {
+        s.parse().unwrap()
+    }
+
+    #[test]
+    fn interface_add_change_flush() {
+        let mut state = BabelState::new();
+
+        state.apply(Event::Interface(InterfaceEvent {
+            kind: EventKind::Add,
+            ifname: "en2".into(),
+            up: true,
+            ipv6: Some(IpAddr::V6(Ipv6Addr::LOCALHOST)),
+            ipv4: Some(Ipv4Addr::new(169, 254, 1, 2)),
+        }));
+        assert_eq!(
+            state.interfaces.get("en2"),
+            Some(&InterfaceState {
+                ifname: "en2".into(),
+                up: true,
+                ipv6: Some(IpAddr::V6(Ipv6Addr::LOCALHOST)),
+                ipv4: Some(Ipv4Addr::new(169, 254, 1, 2)),
+            })
+        );
+
+        state.apply(Event::Interface(InterfaceEvent {
+            kind: EventKind::Change,
+            ifname: "en2".into(),
+            up: false,
+            ipv6: None,
+            ipv4: None,
+        }));
+        assert_eq!(
+            state.interfaces.get("en2"),
+            Some(&InterfaceState {
+                ifname: "en2".into(),
+                up: false,
+                ipv6: None,
+                ipv4: None,
+            })
+        );
+
+        state.apply(Event::Interface(InterfaceEvent {
+            kind: EventKind::Flush,
+            ifname: "en2".into(),
+            up: false,
+            ipv6: None,
+            ipv4: None,
+        }));
+        assert!(!state.interfaces.contains_key("en2"));
+    }
+
+    #[test]
+    fn neighbour_add_and_flush() {
+        let mut state = BabelState::new();
+
+        state.apply(Event::Neighbour(NeighbourEvent {
+            kind: EventKind::Add,
+            handle: 0xabc,
+            address: IpAddr::V6("fe80::1".parse().unwrap()),
+            ifname: "en3".into(),
+            reach: 0x00ff,
+            ureach: 0x000f,
+            rxcost: 96,
+            txcost: 128,
+            rtt_millis: Some(42),
+            rttcost: Some(10),
+            cost: 224,
+        }));
+        assert_eq!(state.neighbours.len(), 1);
+        assert_eq!(state.neighbours.get(&0xabc).unwrap().ifname.as_ref(), "en3");
+
+        state.apply(Event::Neighbour(NeighbourEvent {
+            kind: EventKind::Flush,
+            handle: 0xabc,
+            address: IpAddr::V6("fe80::1".parse().unwrap()),
+            ifname: "en3".into(),
+            reach: 0,
+            ureach: 0,
+            rxcost: 0,
+            txcost: 0,
+            rtt_millis: None,
+            rttcost: None,
+            cost: 0,
+        }));
+        assert!(state.neighbours.is_empty());
+    }
+
+    #[test]
+    fn xroute_change_upserts_by_prefix_pair() {
+        let mut state = BabelState::new();
+
+        state.apply(Event::XRoute(XRouteEvent {
+            kind: EventKind::Add,
+            prefix: net("fde0:20c6:1fa7:ffff::/128"),
+            from: net("::/0"),
+            metric: 256,
+        }));
+        state.apply(Event::XRoute(XRouteEvent {
+            kind: EventKind::Change,
+            prefix: net("fde0:20c6:1fa7:ffff::/128"),
+            from: net("::/0"),
+            metric: 42,
+        }));
+
+        assert_eq!(state.xroutes.len(), 1);
+        assert_eq!(
+            state
+                .xroutes
+                .get(&XRouteKey {
+                    prefix: net("fde0:20c6:1fa7:ffff::/128"),
+                    from: net("::/0"),
+                })
+                .unwrap()
+                .metric,
+            42
+        );
+    }
+
+    #[test]
+    fn route_add_and_flush_by_handle() {
+        let mut state = BabelState::new();
+
+        state.apply(Event::Route(RouteEvent {
+            kind: EventKind::Add,
+            handle: 0xdeadbeef,
+            prefix: net("fde0:20c6:1fa7:ffff::/128"),
+            from: net("::/0"),
+            installed: true,
+            id: Eui64::new(0, 1, 2, 3, 4, 5, 6, 7),
+            metric: 96,
+            refmetric: 96,
+            via: IpAddr::V6("fe80::1234".parse().unwrap()),
+            ifname: "en2".into(),
+        }));
+        assert_eq!(state.routes.len(), 1);
+        assert!(state.routes.get(&0xdeadbeef).unwrap().installed);
+
+        state.apply(Event::Route(RouteEvent {
+            kind: EventKind::Flush,
+            handle: 0xdeadbeef,
+            prefix: net("fde0:20c6:1fa7:ffff::/128"),
+            from: net("::/0"),
+            installed: false,
+            id: Eui64::new(0, 1, 2, 3, 4, 5, 6, 7),
+            metric: 0,
+            refmetric: 0,
+            via: IpAddr::V6("fe80::1234".parse().unwrap()),
+            ifname: "en2".into(),
+        }));
+        assert!(state.routes.is_empty());
+    }
+
+    #[test]
+    fn extend_applies_multiple_events() {
+        let mut state = BabelState::new();
+        state.extend([
+            Event::Interface(InterfaceEvent {
+                kind: EventKind::Add,
+                ifname: "en2".into(),
+                up: true,
+                ipv6: None,
+                ipv4: None,
+            }),
+            Event::XRoute(XRouteEvent {
+                kind: EventKind::Add,
+                prefix: net("fde0:20c6:1fa7:ffff::/128"),
+                from: net("::/0"),
+                metric: 123,
+            }),
+        ]);
+
+        assert_eq!(state.interfaces.len(), 1);
+        assert_eq!(state.xroutes.len(), 1);
+    }
+}

rust/babblerd/src/config.rs

@@ -0,0 +1,141 @@
+//! Process configuration and shared constants for `babblerd`.
+//!
+//! This module centralizes:
+//!
+//! - default runtime paths
+//! - environment variable overrides
+//! - protocol/application constants such as the mesh prefix
+//! - coarse daemon defaults such as the router UDP port
+
+use color_eyre::eyre::{self, eyre};
+use ipnet::Ipv6Net;
+use std::collections::HashSet;
+use std::env;
+use std::net::Ipv6Addr;
+use std::path::{Path, PathBuf};
+
+pub const PUBLIC_SOCKET_PATH_ENV: &str = "BABBLER_SOCKET_PATH";
+pub const NODE_ID_FILE_ENV: &str = "BABBLER_NODE_ID_FILE";
+pub const ROUTER_UDP_PORT_ENV: &str = "BABBLER_ROUTER_UDP_PORT";
+pub const INTERFACE_ALLOWLIST_ENV: &str = "BABBLER_INTERFACE_ALLOWLIST";
+
+pub const DEFAULT_PUBLIC_SOCKET_PATH: &str = {
+    #[cfg(target_os = "macos")]
+    {
+        "/var/run/babbler/babblerd.sock"
+    }
+    #[cfg(target_os = "linux")]
+    {
+        "/run/babbler/babblerd.sock"
+    }
+};
+
+pub const DEFAULT_NODE_ID_FILE: &str = {
+    #[cfg(target_os = "macos")]
+    {
+        "/var/db/babbler/node-id"
+    }
+    #[cfg(target_os = "linux")]
+    {
+        "/var/lib/babbler/node-id"
+    }
+};
+
+// TODO: just picked a random one that didn't seem occupied, there is probably a better way
+//       to do this in the future :)
+pub const DEFAULT_ROUTER_UDP_PORT: u16 = 41897;
+
+pub const PHYSICAL_LINK_MTU: u16 = 1500;
+pub const OUTER_IPV6_HEADER_BYTES: u16 = 40;
+pub const OUTER_UDP_HEADER_BYTES: u16 = 8;
+pub const TUN_MTU: u16 = PHYSICAL_LINK_MTU - OUTER_IPV6_HEADER_BYTES - OUTER_UDP_HEADER_BYTES;
+
+pub const EXO_ULA_PREFIX: Ipv6Net = Ipv6Net::new_assert(
+    // TODO: break out into "fd" for ULA
+    //       e0_20c61fa7 for EXO address-space
+    //       ffff for anything else we want, like maybe versioning and so on (but for now its not used)
+    //
+    // NOTE: spell the hextets explicitly here. A previous `u128` bit-shift
+    // construction accidentally truncated the leading `fde0` and produced
+    // `20c6:1fa7:ffff::/64`, which is not ULA.
+    Ipv6Addr::new(0xfde0, 0x20c6, 0x1fa7, 0xffff, 0, 0, 0, 0),
+    64,
+);
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    pub public_socket_path: PathBuf,
+    pub public_dir: PathBuf,
+    pub node_id_file: PathBuf,
+    pub router_udp_port: u16,
+    pub exo_ula_prefix: Ipv6Net,
+}
+
+impl Config {
+    pub fn from_env() -> eyre::Result<Self> {
+        let public_socket_path = env::var_os(PUBLIC_SOCKET_PATH_ENV)
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from(DEFAULT_PUBLIC_SOCKET_PATH));
+        let Some(public_dir) = public_socket_path.parent().map(Path::to_path_buf) else {
+            return Err(eyre!(
+                "public socket path has no parent directory: {}",
+                public_socket_path.display()
+            ));
+        };
+
+        let node_id_file = env::var_os(NODE_ID_FILE_ENV)
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from(DEFAULT_NODE_ID_FILE));
+
+        let router_udp_port = match env::var(ROUTER_UDP_PORT_ENV) {
+            Ok(raw) => raw
+                .parse::<u16>()
+                .map_err(|e| eyre!("invalid {ROUTER_UDP_PORT_ENV} value {raw:?}: {e}"))?,
+            Err(_) => DEFAULT_ROUTER_UDP_PORT,
+        };
+
+        Ok(Self {
+            public_socket_path,
+            public_dir,
+            node_id_file,
+            router_udp_port,
+            exo_ula_prefix: EXO_ULA_PREFIX,
+        })
+    }
+}
+
+pub fn interface_allowlist_from_env() -> eyre::Result<Option<HashSet<Box<str>>>> {
+    let Ok(raw) = env::var(INTERFACE_ALLOWLIST_ENV) else {
+        return Ok(None);
+    };
+
+    let allowlist = raw
+        .split(',')
+        .map(str::trim)
+        .filter(|name| !name.is_empty())
+        .map(|name| name.into())
+        .collect::<HashSet<Box<str>>>();
+
+    if allowlist.is_empty() {
+        return Err(eyre!(
+            "{INTERFACE_ALLOWLIST_ENV} was set but contained no interface names"
+        ));
+    }
+
+    Ok(Some(allowlist))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::EXO_ULA_PREFIX;
+    use std::net::Ipv6Addr;
+
+    #[test]
+    fn exo_ula_prefix_keeps_fde0_high_bits() {
+        assert_eq!(
+            EXO_ULA_PREFIX.addr(),
+            Ipv6Addr::new(0xfde0, 0x20c6, 0x1fa7, 0xffff, 0, 0, 0, 0)
+        );
+        assert_eq!(EXO_ULA_PREFIX.prefix_len(), 64);
+    }
+}

rust/babblerd/src/daemon.rs

@@ -0,0 +1,381 @@
+use color_eyre::eyre::{Result, WrapErr, eyre};
+use ipnet::Ipv6Net;
+use std::fmt::{Display, Formatter};
+use std::{future::pending, sync::Arc};
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    net::UnixStream,
+    sync::{mpsc, oneshot, watch},
+    task::JoinHandle,
+    time::{Duration, Instant},
+};
+
+use crate::route_ctl;
+use crate::routing_stack::RoutingStack;
+use crate::{babel::BabelState, tun::TunDevice};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ServiceState {
+    Off,
+    Starting,
+    On,
+    Stopping,
+}
+
+impl Display for ServiceState {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Off => write!(f, "off"),
+            Self::Starting => write!(f, "starting"),
+            Self::On => write!(f, "on"),
+            Self::Stopping => write!(f, "stopping"),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DaemonStatus {
+    pub service_state: ServiceState,
+    pub node_id: u64,
+    pub node_addr: Ipv6Net,
+    pub tun_ifname: Arc<str>,
+    // realistically should always have one?? right??
+    pub keepalive_deadline: Option<Instant>,
+    pub last_error: Option<Arc<str>>,
+}
+
+impl DaemonStatus {
+    fn new(node_id: u64, node_addr: Ipv6Net, tun_ifname: Arc<str>) -> Self {
+        Self {
+            service_state: ServiceState::Off,
+            node_id,
+            node_addr,
+            tun_ifname,
+            keepalive_deadline: None,
+            last_error: None,
+        }
+    }
+
+    pub fn render(&self) -> String {
+        let keepalive_remaining_ms = self
+            .keepalive_deadline
+            .and_then(|deadline| deadline.checked_duration_since(Instant::now()))
+            .map(|remaining| remaining.as_millis().to_string())
+            .unwrap_or_else(|| "none".to_owned());
+
+        let mut line = format!(
+            "state {} node_id={:#018x} node_addr={} tun={} keepalive_remaining_ms={}",
+            self.service_state,
+            self.node_id,
+            self.node_addr,
+            self.tun_ifname,
+            keepalive_remaining_ms
+        );
+        if let Some(err) = &self.last_error {
+            line.push_str(&format!(" last_error={err:?}"));
+        }
+        line
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum StackTaskKind {
+    Babel,
+    Watcher,
+    FibPublisher,
+    Dataplane,
+}
+
+#[derive(Debug)]
+pub enum RoutingStackEvent {
+    Exited {
+        kind: StackTaskKind,
+        error: Option<String>,
+    },
+}
+
+#[derive(Debug)]
+enum DaemonCommand {
+    KeepAlive {
+        ttl: Duration,
+        reply: oneshot::Sender<Result<DaemonStatus>>,
+    },
+    GetState {
+        reply: oneshot::Sender<DaemonStatus>,
+    },
+}
+
+#[derive(Clone)]
+pub struct DaemonHandle {
+    send: mpsc::Sender<DaemonCommand>,
+}
+
+impl DaemonHandle {
+    pub async fn keep_alive(&self, ttl: Duration) -> Result<DaemonStatus> {
+        let (reply_send, reply_recv) = oneshot::channel();
+        self.send
+            .send(DaemonCommand::KeepAlive {
+                ttl,
+                reply: reply_send,
+            })
+            .await
+            .map_err(|_| eyre!("daemon core stopped"))?;
+        reply_recv.await.map_err(|_| eyre!("daemon core stopped"))?
+    }
+
+    pub async fn get_state(&self) -> Result<DaemonStatus> {
+        let (reply_send, reply_recv) = oneshot::channel();
+        self.send
+            .send(DaemonCommand::GetState { reply: reply_send })
+            .await
+            .map_err(|_| eyre!("daemon core stopped"))?;
+        reply_recv.await.map_err(|_| eyre!("daemon core stopped"))
+    }
+}
+
+pub struct DaemonCore {
+    status: DaemonStatus,
+    overlay_prefix: Ipv6Net,
+    router_udp_port: u16,
+    _tun: TunDevice,
+    routing_stack: Option<RoutingStack>,
+    babel_state_send: watch::Sender<Arc<BabelState>>,
+    command_recv: mpsc::Receiver<DaemonCommand>,
+    event_send: mpsc::Sender<RoutingStackEvent>,
+    event_recv: mpsc::Receiver<RoutingStackEvent>,
+}
+
+impl DaemonCore {
+    pub fn spawn(
+        node_id: u64,
+        overlay_prefix: Ipv6Net,
+        router_udp_port: u16,
+        node_addr: Ipv6Net,
+        tun: TunDevice,
+        babel_state_send: watch::Sender<Arc<BabelState>>,
+    ) -> (DaemonHandle, JoinHandle<Result<()>>) {
+        let (command_send, command_recv) = mpsc::channel(32);
+        let (event_send, event_recv) = mpsc::channel(8);
+        let status = DaemonStatus::new(node_id, node_addr, Arc::from(tun.ifname().to_owned()));
+
+        let core = Self {
+            status,
+            overlay_prefix,
+            router_udp_port,
+            _tun: tun,
+            routing_stack: None,
+            babel_state_send,
+            command_recv,
+            event_send,
+            event_recv,
+        };
+
+        let handle = DaemonHandle { send: command_send };
+        let task = tokio::spawn(core.run());
+        (handle, task)
+    }
+
+    async fn run(mut self) -> Result<()> {
+        loop {
+            tokio::select! {
+                command = self.command_recv.recv() => {
+                    let Some(command) = command else {
+                        break;
+                    };
+                    self.handle_command(command).await?;
+                }
+                event = self.event_recv.recv() => {
+                    let Some(event) = event else {
+                        break;
+                    };
+                    self.handle_stack_event(event).await?;
+                }
+                _ = lease_timer(self.status.keepalive_deadline) => {
+                    self.handle_lease_expiry().await?;
+                }
+            }
+        }
+
+        self.stop_stack().await?;
+        Ok(())
+    }
+
+    async fn handle_command(&mut self, command: DaemonCommand) -> Result<()> {
+        match command {
+            DaemonCommand::KeepAlive { ttl, reply } => {
+                self.status.keepalive_deadline = Some(Instant::now() + ttl);
+                if self.routing_stack.is_none() {
+                    let result = self.start_stack().await.map(|()| self.status.clone());
+                    let _ = reply.send(result);
+                    return Ok(());
+                }
+                let _ = reply.send(Ok(self.status.clone()));
+                Ok(())
+            }
+            DaemonCommand::GetState { reply } => {
+                let _ = reply.send(self.status.clone());
+                Ok(())
+            }
+        }
+    }
+
+    async fn handle_stack_event(&mut self, event: RoutingStackEvent) -> Result<()> {
+        let RoutingStackEvent::Exited { kind, error } = event;
+        if self.routing_stack.is_none() {
+            return Ok(());
+        }
+
+        tracing::warn!(?kind, ?error, "routing stack task exited");
+        self.status.last_error =
+            Some(Arc::from(error.unwrap_or_else(|| {
+                format!("{kind:?} task exited unexpectedly")
+            })));
+        if let Err(err) = self.stop_stack().await {
+            self.status.last_error = Some(Arc::from(err.to_string()));
+        }
+        Ok(())
+    }
+
+    async fn handle_lease_expiry(&mut self) -> Result<()> {
+        let expired = self
+            .status
+            .keepalive_deadline
+            .is_some_and(|deadline| deadline <= Instant::now());
+        if expired {
+            tracing::info!("keepalive expired, transitioning routing stack off");
+            self.status.keepalive_deadline = None;
+            if let Err(err) = self.stop_stack().await {
+                self.status.last_error = Some(Arc::from(err.to_string()));
+            }
+        }
+        Ok(())
+    }
+
+    async fn start_stack(&mut self) -> Result<()> {
+        if self.routing_stack.is_some() {
+            return Ok(());
+        }
+
+        self.status.service_state = ServiceState::Starting;
+        self.status.last_error = None;
+        match RoutingStack::start(
+            self.status.node_addr,
+            &self._tun,
+            self.router_udp_port,
+            self.babel_state_send.clone(),
+            self.event_send.clone(),
+        ) {
+            Ok(stack) => {
+                self.routing_stack = Some(stack);
+                if let Err(err) = route_ctl::ensure_overlay_route(
+                    self.overlay_prefix,
+                    self.status.tun_ifname.as_ref(),
+                ) {
+                    let _ = self.stop_stack().await;
+                    self.status.service_state = ServiceState::Off;
+                    self.status.last_error = Some(Arc::from(err.to_string()));
+                    return Err(err.into());
+                }
+                self.status.service_state = ServiceState::On;
+                Ok(())
+            }
+            Err(err) => {
+                self.status.service_state = ServiceState::Off;
+                self.status.last_error = Some(Arc::from(err.to_string()));
+                Err(err)
+            }
+        }
+    }
+
+    async fn stop_stack(&mut self) -> Result<()> {
+        if let Err(err) = route_ctl::remove_overlay_route(self.overlay_prefix) {
+            tracing::warn!(error=%err, "failed to remove overlay route");
+        }
+
+        let Some(stack) = self.routing_stack.take() else {
+            self.status.service_state = ServiceState::Off;
+            self.babel_state_send
+                .send_replace(Arc::new(BabelState::new()));
+            return Ok(());
+        };
+
+        self.status.service_state = ServiceState::Stopping;
+        let stop_result = stack.stop().await;
+        self.babel_state_send
+            .send_replace(Arc::new(BabelState::new()));
+        self.status.service_state = ServiceState::Off;
+        if let Err(err) = stop_result {
+            self.status.last_error = Some(Arc::from(err.to_string()));
+            return Err(err);
+        }
+        Ok(())
+    }
+}
+
+async fn lease_timer(deadline: Option<Instant>) {
+    if let Some(deadline) = deadline {
+        tokio::time::sleep_until(deadline).await;
+    } else {
+        pending::<()>().await;
+    }
+}
+
+pub async fn handle_client(sock: UnixStream, daemon: DaemonHandle) {
+    tracing::info!("new socket conn");
+    let (reader, mut write) = sock.into_split();
+    let mut reader = BufReader::new(reader).lines();
+
+    if let Ok(state) = daemon.get_state().await {
+        let _ = write
+            .write_all(format!("{}\n", state.render()).as_bytes())
+            .await;
+    }
+
+    loop {
+        let Ok(Some(line)) = reader.next_line().await else {
+            break;
+        };
+        let trimmed = line.trim();
+        if trimmed.is_empty() {
+            continue;
+        }
+
+        let response = match handle_command_line(trimmed, &daemon).await {
+            Ok(response) => response,
+            Err(err) => format!("error {err}"),
+        };
+
+        if let Err(err) = write.write_all(format!("{response}\n").as_bytes()).await {
+            tracing::warn!(error=%err, "failed to write command response");
+            break;
+        }
+    }
+
+    tracing::info!("closing socket conn");
+    let _ = write.shutdown().await;
+}
+
+async fn handle_command_line(line: &str, daemon: &DaemonHandle) -> Result<String> {
+    let mut parts = line.split_whitespace();
+    let Some(command) = parts.next() else {
+        return Ok("error empty-command".to_owned());
+    };
+
+    match command {
+        "get-state" => Ok(daemon.get_state().await?.render()),
+        "keepalive" => {
+            let Some(ttl_ms) = parts.next() else {
+                return Err(eyre!("keepalive requires ttl_ms"));
+            };
+            let ttl_ms = ttl_ms
+                .parse::<u64>()
+                .wrap_err_with(|| format!("invalid ttl_ms: {ttl_ms:?}"))?;
+            Ok(daemon
+                .keep_alive(Duration::from_millis(ttl_ms))
+                .await?
+                .render())
+        }
+        "help" => Ok("commands: get-state | keepalive <ttl_ms>".to_owned()),
+        other => Err(eyre!("unknown command: {other}")),
+    }
+}

rust/babblerd/src/fib.rs

@@ -0,0 +1,390 @@
+//! Immutable forwarding snapshots derived from [`crate::babel::BabelState`].
+//!
+//! `BabelState` mirrors the control-plane view emitted by `babeld`.
+//! `FibSnapshot` is the reduced dataplane view:
+//!
+//! - exact-match IPv6 host routes only for now,
+//! - admitted interface ownership alongside those routes,
+//! - one immutable snapshot swapped wholesale into the dataplane,
+//! - keyed for fast lookup rather than protocol fidelity.
+//!
+//! The v1 forwarding model is intentionally narrow:
+//!
+//! - local addresses are explicit inputs, not inferred from every xroute,
+//! - only interfaces with a live Babel neighbour are exposed to the dataplane,
+//! - only installed IPv6 `/128` routes are considered,
+//! - only destination-based forwarding is modeled,
+//! - routes with non-link-local next hops are ignored.
+
+use std::net::Ipv6Addr;
+
+use ahash::RandomState;
+use hashbrown::{HashMap, HashSet, hash_map::Entry};
+use ipnet::{IpNet, Ipv6Net};
+
+use crate::babel::BabelState;
+use crate::babel::state::RouteState;
+
+pub type HostKey = u128;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FibEntry {
+    pub next_hop_ll: Ipv6Addr,
+    pub ifname: Box<str>,
+    pub mtu: u16,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FibSnapshot {
+    pub locals: HashSet<HostKey, RandomState>,
+    pub admitted_interfaces: HashSet<Box<str>, RandomState>,
+    pub routes: HashMap<HostKey, FibEntry, RandomState>,
+}
+
+#[derive(Debug, Clone)]
+pub struct FibBuilder {
+    local_addrs: Vec<Ipv6Addr>,
+    route_mtu: u16,
+}
+
+impl FibBuilder {
+    pub fn new<I>(local_addrs: I, route_mtu: u16) -> Self
+    where
+        I: IntoIterator<Item = Ipv6Addr>,
+    {
+        Self {
+            local_addrs: local_addrs.into_iter().collect(),
+            route_mtu,
+        }
+    }
+
+    pub fn derive(&self, state: &BabelState) -> FibSnapshot {
+        let mut locals =
+            HashSet::with_capacity_and_hasher(self.local_addrs.len(), RandomState::new());
+        for addr in &self.local_addrs {
+            locals.insert(host_key(*addr));
+        }
+
+        let up_interfaces: HashSet<&str, RandomState> = state
+            .interfaces
+            .values()
+            .filter(|interface| interface.up)
+            .map(|interface| interface.ifname.as_ref())
+            .collect::<HashSet<_, _>>();
+
+        let mut admitted_interfaces = HashSet::with_hasher(RandomState::new());
+        for neighbour in state.neighbours.values() {
+            if up_interfaces.contains(neighbour.ifname.as_ref()) {
+                admitted_interfaces.insert(neighbour.ifname.clone());
+            }
+        }
+
+        let mut routes = HashMap::with_hasher(RandomState::new());
+        let mut route_scores = HashMap::with_hasher(RandomState::new());
+
+        let mut candidates: Vec<&RouteState> = state.routes.values().collect();
+        candidates.sort_by(|left, right| {
+            left.ifname
+                .cmp(&right.ifname)
+                .then_with(|| left.prefix.to_string().cmp(&right.prefix.to_string()))
+                .then_with(|| left.metric.cmp(&right.metric))
+                .then_with(|| left.refmetric.cmp(&right.refmetric))
+                .then_with(|| left.handle.cmp(&right.handle))
+        });
+
+        for route in candidates {
+            let Some((dst, next_hop_ll)) = route_to_host(route) else {
+                continue;
+            };
+            if !admitted_interfaces.contains(route.ifname.as_ref()) {
+                continue;
+            }
+            if locals.contains(&dst) {
+                continue;
+            }
+
+            let candidate = FibEntry {
+                next_hop_ll,
+                ifname: route.ifname.clone(),
+                mtu: self.route_mtu,
+            };
+            let candidate_score = (route.metric, route.refmetric, route.handle);
+
+            match routes.entry(dst) {
+                Entry::Vacant(slot) => {
+                    slot.insert(candidate);
+                    route_scores.insert(dst, candidate_score);
+                }
+                Entry::Occupied(mut slot) => {
+                    let Some(existing_score) = route_scores.get(&dst).copied() else {
+                        slot.insert(candidate);
+                        route_scores.insert(dst, candidate_score);
+                        continue;
+                    };
+
+                    if candidate_score < existing_score {
+                        slot.insert(candidate);
+                        route_scores.insert(dst, candidate_score);
+                    }
+                }
+            }
+        }
+
+        FibSnapshot {
+            locals,
+            admitted_interfaces,
+            routes,
+        }
+    }
+}
+
+impl FibSnapshot {
+    pub fn empty() -> Self {
+        Self {
+            locals: HashSet::with_hasher(RandomState::new()),
+            admitted_interfaces: HashSet::with_hasher(RandomState::new()),
+            routes: HashMap::with_hasher(RandomState::new()),
+        }
+    }
+
+    pub fn from_node_addr(node_addr: Ipv6Net, state: &BabelState, route_mtu: u16) -> Self {
+        FibBuilder::new([node_addr.addr()], route_mtu).derive(state)
+    }
+
+    pub fn is_local(&self, addr: Ipv6Addr) -> bool {
+        self.locals.contains(&host_key(addr))
+    }
+
+    pub fn lookup(&self, addr: Ipv6Addr) -> Option<&FibEntry> {
+        self.routes.get(&host_key(addr))
+    }
+}
+
+pub fn host_key(addr: Ipv6Addr) -> HostKey {
+    u128::from(addr)
+}
+
+fn route_to_host(route: &RouteState) -> Option<(HostKey, Ipv6Addr)> {
+    if !route.installed {
+        return None;
+    }
+
+    let IpNet::V6(prefix) = route.prefix else {
+        return None;
+    };
+    if prefix.prefix_len() != 128 {
+        return None;
+    }
+
+    let IpNet::V6(from) = route.from else {
+        return None;
+    };
+    if from.prefix_len() != 0 || from.addr() != Ipv6Addr::UNSPECIFIED {
+        return None;
+    }
+
+    let std::net::IpAddr::V6(via) = route.via else {
+        return None;
+    };
+    if !via.is_unicast_link_local() {
+        return None;
+    }
+
+    Some((host_key(prefix.addr()), via))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::net::{IpAddr, Ipv6Addr};
+
+    use crate::babel::Eui64;
+    use crate::babel::line::{Event, EventKind, InterfaceEvent, NeighbourEvent, RouteEvent};
+    use crate::babel::state::BabelState;
+
+    use super::FibBuilder;
+
+    fn route(
+        handle: u64,
+        prefix: &str,
+        from: &str,
+        installed: bool,
+        via: Ipv6Addr,
+        ifname: &str,
+        metric: u32,
+        refmetric: u32,
+    ) -> Event {
+        Event::Route(RouteEvent {
+            kind: EventKind::Add,
+            handle,
+            prefix: prefix.parse().unwrap(),
+            from: from.parse().unwrap(),
+            installed,
+            id: Eui64::new(0, 1, 2, 3, 4, 5, 6, 7),
+            metric,
+            refmetric,
+            via: IpAddr::V6(via),
+            ifname: ifname.into(),
+        })
+    }
+
+    fn interface(ifname: &str, up: bool) -> Event {
+        Event::Interface(InterfaceEvent {
+            kind: EventKind::Add,
+            ifname: ifname.into(),
+            up,
+            ipv6: None,
+            ipv4: None,
+        })
+    }
+
+    fn neighbour(handle: u64, ifname: &str, address: &str) -> Event {
+        Event::Neighbour(NeighbourEvent {
+            kind: EventKind::Add,
+            handle,
+            address: IpAddr::V6(address.parse().unwrap()),
+            ifname: ifname.into(),
+            reach: 0xffff,
+            ureach: 0xffff,
+            rxcost: 96,
+            txcost: 96,
+            rtt_millis: Some(1),
+            rttcost: Some(0),
+            cost: 96,
+        })
+    }
+
+    #[test]
+    fn derives_local_and_host_routes() {
+        let mut state = BabelState::new();
+        state.apply(interface("en2", true));
+        state.apply(neighbour(1, "en2", "fe80::1"));
+        state.apply(route(
+            1,
+            "fde0::1234/128",
+            "::/0",
+            true,
+            "fe80::1".parse().unwrap(),
+            "en2",
+            96,
+            32,
+        ));
+
+        let fib = FibBuilder::new(["fde0::1".parse().unwrap()], 1452).derive(&state);
+
+        assert!(fib.is_local("fde0::1".parse().unwrap()));
+        assert!(fib.admitted_interfaces.contains("en2"));
+        let entry = fib.lookup("fde0::1234".parse().unwrap()).unwrap();
+        assert_eq!(entry.next_hop_ll, "fe80::1".parse::<Ipv6Addr>().unwrap());
+        assert_eq!(entry.ifname.as_ref(), "en2");
+        assert_eq!(entry.mtu, 1452);
+    }
+
+    #[test]
+    fn skips_non_installed_or_non_host_routes() {
+        let mut state = BabelState::new();
+        state.apply(interface("en2", true));
+        state.apply(interface("en3", true));
+        state.apply(neighbour(1, "en2", "fe80::1"));
+        state.apply(neighbour(2, "en3", "fe80::2"));
+        state.apply(route(
+            1,
+            "fde0::abcd/128",
+            "::/0",
+            false,
+            "fe80::1".parse().unwrap(),
+            "en2",
+            96,
+            32,
+        ));
+        state.apply(route(
+            2,
+            "fde0::/64",
+            "::/0",
+            true,
+            "fe80::2".parse().unwrap(),
+            "en3",
+            96,
+            32,
+        ));
+
+        let fib = FibBuilder::new(["fde0::1".parse().unwrap()], 1452).derive(&state);
+        assert!(fib.routes.is_empty());
+    }
+
+    #[test]
+    fn prefers_lower_metric_when_multiple_installed_routes_exist() {
+        let mut state = BabelState::new();
+        state.apply(interface("en2", true));
+        state.apply(interface("en3", true));
+        state.apply(neighbour(1, "en2", "fe80::1"));
+        state.apply(neighbour(2, "en3", "fe80::2"));
+        state.apply(route(
+            1,
+            "fde0::beef/128",
+            "::/0",
+            true,
+            "fe80::1".parse().unwrap(),
+            "en2",
+            200,
+            20,
+        ));
+        state.apply(route(
+            2,
+            "fde0::beef/128",
+            "::/0",
+            true,
+            "fe80::2".parse().unwrap(),
+            "en3",
+            100,
+            10,
+        ));
+
+        let fib = FibBuilder::new(["fde0::1".parse().unwrap()], 1452).derive(&state);
+        let entry = fib.lookup("fde0::beef".parse().unwrap()).unwrap();
+        assert_eq!(entry.next_hop_ll, "fe80::2".parse::<Ipv6Addr>().unwrap());
+        assert_eq!(entry.ifname.as_ref(), "en3");
+    }
+
+    #[test]
+    fn skips_routes_for_interfaces_without_live_neighbours() {
+        let mut state = BabelState::new();
+        state.apply(interface("en2", true));
+        state.apply(route(
+            1,
+            "fde0::cafe/128",
+            "::/0",
+            true,
+            "fe80::1".parse().unwrap(),
+            "en2",
+            96,
+            32,
+        ));
+
+        let fib = FibBuilder::new(["fde0::1".parse().unwrap()], 1452).derive(&state);
+
+        assert!(fib.admitted_interfaces.is_empty());
+        assert!(fib.lookup("fde0::cafe".parse().unwrap()).is_none());
+    }
+
+    #[test]
+    fn skips_neighbour_interfaces_that_are_not_up() {
+        let mut state = BabelState::new();
+        state.apply(interface("en2", false));
+        state.apply(neighbour(1, "en2", "fe80::1"));
+        state.apply(route(
+            1,
+            "fde0::cafe/128",
+            "::/0",
+            true,
+            "fe80::1".parse().unwrap(),
+            "en2",
+            96,
+            32,
+        ));
+
+        let fib = FibBuilder::new(["fde0::1".parse().unwrap()], 1452).derive(&state);
+
+        assert!(fib.admitted_interfaces.is_empty());
+        assert!(fib.lookup("fde0::cafe".parse().unwrap()).is_none());
+    }
+}

rust/babblerd/src/identity.rs

@@ -0,0 +1,154 @@
+//! Persistent node identity for `babblerd`.
+//!
+//! The node ID occupies the full low 64 bits of the EXO ULA space.
+
+use std::fs::{self, OpenOptions};
+use std::io::Write;
+use std::os::unix::fs::{MetadataExt, PermissionsExt};
+use std::path::Path;
+
+use color_eyre::eyre::{self, WrapErr, eyre};
+use ipnet::Ipv6Net;
+use nix::unistd::geteuid;
+use std::net::Ipv6Addr;
+
+pub fn load_or_create_node_id(path: &Path) -> eyre::Result<u64> {
+    match read_node_id(path) {
+        Ok(node_id) => Ok(node_id),
+        Err(err) if is_not_found(&err) => create_node_id(path),
+        Err(err) => Err(err),
+    }
+}
+
+pub fn node_addr(prefix: Ipv6Net, node_id: u64) -> eyre::Result<Ipv6Net> {
+    if prefix.prefix_len() != 64 {
+        return Err(eyre!(
+            "expected EXO ULA prefix to be /64, got {prefix} with /{}",
+            prefix.prefix_len()
+        ));
+    }
+    Ok(Ipv6Net::new_assert(
+        Ipv6Addr::from_bits(prefix.trunc().addr().to_bits() | u128::from(node_id)),
+        128,
+    ))
+}
+
+fn create_node_id(path: &Path) -> eyre::Result<u64> {
+    let Some(parent) = path.parent() else {
+        return Err(eyre!(
+            "node id file has no parent directory: {}",
+            path.display()
+        ));
+    };
+    fs::create_dir_all(parent)
+        .wrap_err_with(|| format!("creating node id directory {}", parent.display()))?;
+
+    let node_id = generate_node_id();
+    let mut file = match OpenOptions::new().write(true).create_new(true).open(path) {
+        Ok(file) => file,
+        Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {
+            return read_node_id(path);
+        }
+        Err(err) => {
+            return Err(err).wrap_err_with(|| format!("creating node id file {}", path.display()));
+        }
+    };
+
+    file.set_permissions(fs::Permissions::from_mode(0o600))
+        .wrap_err_with(|| format!("setting permissions on {}", path.display()))?;
+
+    writeln!(file, "{node_id:016x}")
+        .wrap_err_with(|| format!("writing node id file {}", path.display()))?;
+    file.sync_all()
+        .wrap_err_with(|| format!("syncing node id file {}", path.display()))?;
+    drop(file);
+
+    read_node_id(path)
+}
+
+fn read_node_id(path: &Path) -> eyre::Result<u64> {
+    let metadata =
+        fs::metadata(path).wrap_err_with(|| format!("reading metadata for {}", path.display()))?;
+    ensure_owner(path, &metadata)?;
+
+    let raw = fs::read_to_string(path)
+        .wrap_err_with(|| format!("reading node id file {}", path.display()))?;
+    let trimmed = raw.trim();
+    if trimmed.is_empty() {
+        return Err(eyre!("node id file is empty: {}", path.display()));
+    }
+
+    let node_id = u64::from_str_radix(trimmed.trim_start_matches("0x"), 16)
+        .wrap_err_with(|| format!("invalid node id in {}: {:?}", path.display(), trimmed))?;
+    Ok(node_id)
+}
+
+fn ensure_owner(path: &Path, metadata: &fs::Metadata) -> eyre::Result<()> {
+    let expected_uid = geteuid().as_raw();
+    let actual_uid = metadata.uid();
+    if actual_uid != expected_uid {
+        return Err(eyre!(
+            "node id file {} is owned by uid {}, expected {}",
+            path.display(),
+            actual_uid,
+            expected_uid
+        ));
+    }
+    Ok(())
+}
+
+fn generate_node_id() -> u64 {
+    rand::random::<u64>()
+}
+
+fn is_not_found(err: &eyre::Report) -> bool {
+    err.downcast_ref::<std::io::Error>()
+        .is_some_and(|e| e.kind() == std::io::ErrorKind::NotFound)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn temp_path(name: &str) -> std::path::PathBuf {
+        let nonce = rand::random::<u64>();
+        std::env::temp_dir().join(format!(
+            "babblerd-identity-{name}-{}-{nonce}",
+            std::process::id()
+        ))
+    }
+
+    #[test]
+    fn creates_and_reloads_same_node_id() {
+        let dir = temp_path("create");
+        let path = dir.join("node-id");
+
+        let first = load_or_create_node_id(&path).expect("create node id");
+        let second = load_or_create_node_id(&path).expect("reload node id");
+
+        assert_eq!(first, second);
+
+        let _ = fs::remove_file(&path);
+        let _ = fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn node_addr_uses_full_low_64_bits() {
+        let addr = node_addr(
+            Ipv6Net::new_assert(
+                Ipv6Addr::new(0xfde0, 0x20c6, 0x1fa7, 0xffff, 0, 0, 0, 0),
+                64,
+            ),
+            0x1234_5678_9abc_def0,
+        )
+        .expect("node address should be constructed");
+
+        assert_eq!(addr.prefix_len(), 128);
+        assert_eq!(
+            addr.addr(),
+            Ipv6Addr::new(
+                0xfde0, 0x20c6, 0x1fa7, 0xffff, 0x1234, 0x5678, 0x9abc, 0xdef0,
+            )
+        );
+    }
+}

rust/babblerd/src/lib.rs

@@ -0,0 +1,335 @@
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+compile_error!("babblerd is mac/linux-only");
+
+pub mod babel;
+pub mod config;
+pub mod daemon;
+pub mod dataplane;
+pub mod fib;
+pub mod identity;
+pub mod profiling;
+pub(crate) mod route_ctl;
+pub mod routing_stack;
+pub mod tun;
+
+pub use babel::babel;
+pub use config::EXO_ULA_PREFIX as PREFIX;
+pub use error::{BabbleError, Result};
+pub use if_watcher::watch;
+pub mod error {
+    use std::io;
+    use thiserror::Error;
+
+    pub type Result<T> = core::result::Result<T, BabbleError>;
+
+    #[derive(Error, Debug)]
+    pub enum BabbleError {
+        #[error("An IO error occurred: {0}")]
+        Io(#[from] io::Error),
+        #[error("Unspecified error")]
+        Unspecified,
+        #[error("Babeld crashed unexpectedly with code: {0:?}")]
+        BabeldCrashed(Option<i32>),
+        #[error("Failed to set IP address")]
+        FailedToSetIp,
+        #[error("Other error: {0}")]
+        Other(String),
+    }
+}
+pub mod if_watcher {
+    #[cfg(target_os = "linux")]
+    use std::path::PathBuf;
+    use std::{collections::HashSet, net::IpAddr};
+
+    use futures_lite::StreamExt;
+    use n0_watcher::Watcher;
+    use netwatch::interfaces::{Interface, IpNet};
+    use tokio::sync::mpsc;
+
+    use crate::config::{EXO_ULA_PREFIX, PHYSICAL_LINK_MTU, interface_allowlist_from_env};
+    use crate::ip_manager::remove_ip;
+    use crate::{BabbleError, Result, babel::Babble};
+
+    pub const LOCALHOST_INTERFACE_NAMES: [&'static str; 2] = ["lo", "lo0"];
+
+    trait IfaceExt {
+        fn has_link_local_v6(&self) -> bool;
+        fn has_required_mtu(&self) -> bool;
+        fn is_real_interface(&self) -> bool;
+        fn will_babel(&self) -> bool;
+    }
+    impl IfaceExt for Interface {
+        fn will_babel(&self) -> bool {
+            self.has_link_local_v6()
+                && self.has_required_mtu()
+                && self.is_real_interface()
+                && self.is_up()
+        }
+
+        fn has_link_local_v6(&self) -> bool {
+            let mut has = false;
+            for addr in self.addrs() {
+                let IpAddr::V6(a) = addr.addr() else {
+                    continue;
+                };
+                if a.is_unicast_link_local() {
+                    has = true;
+                    break;
+                }
+            }
+            has
+        }
+
+        fn has_required_mtu(&self) -> bool {
+            let Some(mtu) = interface_mtu(self.name()) else {
+                tracing::debug!(
+                    "skipping interface {} because MTU could not be determined",
+                    self.name()
+                );
+                return false;
+            };
+            if mtu < u32::from(PHYSICAL_LINK_MTU) {
+                tracing::debug!(
+                    "skipping interface {} because mtu {} is below required {}",
+                    self.name(),
+                    mtu,
+                    PHYSICAL_LINK_MTU
+                );
+                return false;
+            }
+            true
+        }
+
+        fn is_real_interface(&self) -> bool {
+            // macos is weird. en0 & en1 are ethernet & wifi (varies which is which by device). en3+ is thunderbolt, but at some point becomes usb ethernet.
+            if self.name().strip_prefix("en").is_none()
+            //.and_then(|s| s.parse::<u8>().ok())
+            //.is_none_or(|_n| false)
+            {
+                return false;
+            }
+            #[cfg(target_os = "linux")]
+            {
+                if !PathBuf::from(format!("/sys/class/net/{}/device", self.name())).exists() {
+                    tracing::debug!(
+                        "skipping interface {} as it doesn't correspond to a physical link",
+                        self.name()
+                    );
+                    return false;
+                }
+                let dev_type_path = PathBuf::from(format!("/sys/class/net/{}/type", self.name()));
+                if !dev_type_path.exists() {
+                    tracing::debug!(
+                        "skipping interface {} with no type file at {:?}",
+                        self.name(),
+                        dev_type_path.to_str()
+                    );
+                    return false;
+                }
+                let Ok(dev_type) = std::fs::read_to_string(dev_type_path) else {
+                    return false;
+                };
+                if dev_type.trim() != "1" {
+                    tracing::debug!(
+                        "skipping interface {} with type {:?}",
+                        self.name(),
+                        dev_type
+                    );
+                    return false;
+                }
+            }
+            true
+        }
+    }
+
+    fn interface_mtu(name: &str) -> Option<u32> {
+        netdev::get_interfaces()
+            .into_iter()
+            .find(|iface| iface.name == name)
+            .and_then(|iface| iface.mtu)
+    }
+
+    #[tracing::instrument(skip(send))]
+    pub async fn watch(send: mpsc::Sender<Babble>) -> Result<()> {
+        let mut ready_ifaces = HashSet::new();
+        let interface_allowlist = interface_allowlist_from_env()
+            .map_err(|e| BabbleError::Other(format!("invalid interface allowlist: {e}")))?;
+
+        if let Some(allowlist) = &interface_allowlist {
+            tracing::info!(?allowlist, "interface allowlist active");
+        }
+
+        tracing::info!("starting interface monitor");
+        let mon = netwatch::netmon::Monitor::new()
+            .await
+            .map_err(|_| BabbleError::Unspecified)?;
+
+        // TODD: this should never really be a thing thats the case, BUT I like the idea of having
+        //       "heuristic" scripts that can help resolve issues but not necessarily gurantee success;
+        //       I like the idea of generalising this concept into a framework where we have "heuristic tasks"
+        //       that run to aid in tyring to fix some system ale-ment or whatever
+        //
+        // one-shot cleanup:
+        // - remove any stale app-prefix addresses from lo0
+        // - remove any app-prefix addresses that accidentally landed on physical links
+        {
+            let state = mon.interface_state();
+            for iface in state.peek().interfaces.values() {
+                let cleanup_target =
+                    LOCALHOST_INTERFACE_NAMES.contains(&iface.name()) || iface.is_real_interface();
+                if !cleanup_target {
+                    continue;
+                }
+                for addr in iface.addrs() {
+                    if let IpNet::V6 { net: v6, .. } = addr
+                        && EXO_ULA_PREFIX.contains(&v6.addr())
+                    {
+                        tracing::info!("removing stale app ip {v6} from {}", iface.name());
+                        if let Err(e) = remove_ip(v6, iface).await {
+                            tracing::warn!(%e, "failed to remove stale app ip");
+                        }
+                    }
+                }
+            }
+        }
+
+        // stream updates
+        let mut mon_stream = mon.interface_state().stream();
+        while let Some(s) = mon_stream.next().await {
+            for iface in s.interfaces.values() {
+                if let Some(allowlist) = &interface_allowlist
+                    && !allowlist.contains(iface.name())
+                {
+                    tracing::debug!(
+                        "skipping interface {} because it is not in {}",
+                        iface.name(),
+                        crate::config::INTERFACE_ALLOWLIST_ENV
+                    );
+                    continue;
+                }
+
+                if !iface.is_real_interface() {
+                    continue;
+                }
+
+                // physical links should not carry babbler application-space addresses
+                for addr in iface.addrs() {
+                    if let IpNet::V6 { net: v6, .. } = addr
+                        && EXO_ULA_PREFIX.contains(&v6.addr())
+                    {
+                        tracing::info!("removing app ip {v6} from {}", iface.name());
+                        if let Err(e) = remove_ip(v6, iface).await {
+                            tracing::warn!(%e, "failed to remove ip");
+                        }
+                    }
+                }
+                if !iface.will_babel() {
+                    continue;
+                }
+                if ready_ifaces.insert(iface.name().to_owned()) {
+                    tracing::info!("telling babeld to watch {}", iface.name());
+                    let Ok(()) = send.send(Babble::AddIface(iface.name().into())).await else {
+                        return Ok(());
+                    };
+                }
+            }
+        }
+        tracing::info!("stopping interface monitor");
+
+        Ok(())
+    }
+}
+
+pub(crate) mod ip_manager {
+    pub use sys::add_ip;
+    pub use sys::remove_ip;
+
+    #[cfg(target_os = "linux")]
+    mod sys {
+        use ipnet::Ipv6Net;
+        use netwatch::interfaces::Interface;
+
+        use crate::{BabbleError, Result};
+        use tokio::process::Command;
+
+        #[tracing::instrument]
+        pub async fn add_ip(subnet: Ipv6Net, iface: &Interface) -> Result<()> {
+            let out = Command::new("ip")
+                .arg("addr")
+                .arg("add")
+                .arg(format!("{subnet}"))
+                .arg("dev")
+                .arg(iface.name())
+                .output()
+                .await?;
+            if out.status.success() {
+                Ok(())
+            } else {
+                Err(BabbleError::FailedToSetIp)
+            }
+        }
+
+        #[tracing::instrument]
+        pub async fn remove_ip(v6: Ipv6Net, iface: &Interface) -> Result<()> {
+            let out = Command::new("ip")
+                .arg("addr")
+                .arg("del")
+                .arg(format!("{v6}"))
+                .arg("dev")
+                .arg(iface.name())
+                .output()
+                .await?;
+            if out.status.success() {
+                Ok(())
+            } else {
+                let std_err = String::from_utf8_lossy(&out.stdout);
+                tracing::debug!(%std_err);
+                Err(BabbleError::FailedToSetIp)
+            }
+        }
+    }
+
+    #[cfg(target_os = "macos")]
+    mod sys {
+        use ipnet::Ipv6Net;
+        use netwatch::interfaces::Interface;
+
+        use crate::BabbleError;
+        use crate::Result;
+        use tokio::process::Command;
+
+        #[tracing::instrument]
+        pub async fn add_ip(subnet: Ipv6Net, iface: &Interface) -> Result<()> {
+            let out = Command::new("ifconfig")
+                .arg(iface.name())
+                .arg("inet6")
+                .arg(format!("{subnet}"))
+                .arg("add")
+                .output()
+                .await?;
+            if out.status.success() {
+                Ok(())
+            } else {
+                Err(BabbleError::FailedToSetIp)
+            }
+        }
+
+        #[tracing::instrument]
+        pub async fn remove_ip(v6: Ipv6Net, iface: &Interface) -> Result<()> {
+            let out = Command::new("ifconfig")
+                .arg(iface.name())
+                .arg("inet6")
+                .arg(format!("{v6}"))
+                .arg("delete")
+                .output()
+                .await?;
+            if out.status.success() {
+                Ok(())
+            } else {
+                let std_err = String::from_utf8_lossy(&out.stdout);
+                tracing::debug!(%std_err);
+                Err(BabbleError::FailedToSetIp)
+            }
+        }
+    }
+}

rust/babblerd/src/main.rs

@@ -0,0 +1,195 @@
+// Major TODO: at some point don't call it "babbler" because that is a silly name that makes no sense
+//             but this is at the very bottom of my concerns right now :)
+
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+compile_error!("babblerd is mac/linux-only");
+
+use std::{fs::Permissions, io, os::unix::fs::PermissionsExt, sync::Arc};
+
+use babblerd::{babel::BabelState, config::Config, daemon, identity, tun::TunDevice};
+use color_eyre::eyre::{self, WrapErr, eyre};
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    net::UnixListener,
+    net::UnixStream,
+    signal,
+    sync::watch,
+    task::JoinSet,
+    time::{Duration, sleep},
+};
+
+const INTERNAL_KEEPALIVE_TTL_MS: u64 = 30_000;
+const INTERNAL_KEEPALIVE_INTERVAL_MS: u64 = 10_000;
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+    color_eyre::install()?;
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .init();
+    let config = Config::from_env()?;
+
+    // cleanup old  public socket path
+    match std::fs::remove_file(&config.public_socket_path) {
+        Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.into()),
+        Ok(()) => {
+            tracing::info!(
+                "cleaned up old file at {}",
+                config.public_socket_path.display()
+            );
+        }
+        _ => {}
+    }
+
+    // create new public directory
+    std::fs::create_dir_all(&config.public_dir)?;
+    if let Err(e) = std::fs::set_permissions(&config.public_dir, Permissions::from_mode(0o0755)) {
+        if e.kind() == io::ErrorKind::PermissionDenied {
+            return Err(eyre!(
+                "Insufficient permissions to run daemon -- did you forget sudo?"
+            ));
+        }
+        return Err(e.into());
+    }
+
+    let res = inner_main(&config).await;
+    let _ = std::fs::remove_file(&config.public_socket_path);
+    res
+}
+
+async fn inner_main(config: &Config) -> eyre::Result<()> {
+    let node_id = identity::load_or_create_node_id(&config.node_id_file)?;
+    let node_addr = identity::node_addr(config.exo_ula_prefix, node_id)?;
+    let tun = TunDevice::create(node_addr.addr()).wrap_err("creating tun for node address")?;
+
+    tracing::info!("creating socket at {}", config.public_socket_path.display());
+    tracing::info!(
+        "router defaults: udp_port={} node_id_file={} app_prefix={} node_id={:#018x} node_addr={} tun={}",
+        config.router_udp_port,
+        config.node_id_file.display(),
+        config.exo_ula_prefix,
+        node_id,
+        node_addr,
+        tun.ifname(),
+    );
+
+    let public_socket = UnixListener::bind(&config.public_socket_path)?;
+
+    // make our socket world accessible
+    std::fs::set_permissions(&config.public_socket_path, Permissions::from_mode(0o0666))?;
+
+    let (babel_state_send, _) = watch::channel(Arc::new(BabelState::new()));
+    let (daemon, mut core_task) = daemon::DaemonCore::spawn(
+        node_id,
+        config.exo_ula_prefix,
+        config.router_udp_port,
+        node_addr,
+        tun,
+        babel_state_send,
+    );
+    // TEMP: keep the daemon alive without an external client until the real
+    // frontend/test harness exists. This should be removed later.
+    let mut internal_keepalive =
+        tokio::spawn(internal_keepalive_client(config.public_socket_path.clone()));
+    let mut listeners = JoinSet::new();
+
+    loop {
+        tokio::select! {
+            sig = signal::ctrl_c() => {
+                sig?;
+                internal_keepalive.abort();
+                let _ = (&mut internal_keepalive).await;
+                listeners.abort_all();
+                while let Some(res) = listeners.join_next().await {
+                    res.wrap_err("while ctrl-c")?;
+                }
+                drop(daemon);
+                core_task.await??;
+                break;
+            }
+            sock = public_socket.accept() => {
+                let sock = sock?.0;
+                listeners.spawn(daemon::handle_client(sock, daemon.clone()));
+            }
+            res = &mut core_task => {
+                res??;
+                internal_keepalive.abort();
+                let _ = (&mut internal_keepalive).await;
+                listeners.abort_all();
+                while let Some(res2) = listeners.join_next().await {
+                    res2.wrap_err("while closing daemon core")?;
+                }
+                break;
+            }
+            res = &mut internal_keepalive => {
+                return Err(eyre!("internal keepalive client exited unexpectedly: {res:?}"));
+            }
+            next_join_result = listeners.join_next(), if !listeners.is_empty() => {
+                next_join_result.expect("checked")?;
+                tracing::info!("dropped a listener");
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn internal_keepalive_client(socket_path: std::path::PathBuf) {
+    loop {
+        match UnixStream::connect(&socket_path).await {
+            Ok(stream) => {
+                tracing::info!(
+                    socket=%socket_path.display(),
+                    "internal keepalive client connected"
+                );
+                let (reader, mut writer) = stream.into_split();
+                let mut reader = BufReader::new(reader).lines();
+
+                match reader.next_line().await {
+                    Ok(Some(line)) => {
+                        tracing::debug!(?line, "internal keepalive initial state");
+                    }
+                    Ok(None) => {
+                        tracing::warn!("internal keepalive connection closed before initial state");
+                        sleep(Duration::from_secs(1)).await;
+                        continue;
+                    }
+                    Err(err) => {
+                        tracing::warn!(error=%err, "internal keepalive failed to read initial state");
+                        sleep(Duration::from_secs(1)).await;
+                        continue;
+                    }
+                }
+
+                loop {
+                    let command = format!("keepalive {INTERNAL_KEEPALIVE_TTL_MS}\n");
+                    if let Err(err) = writer.write_all(command.as_bytes()).await {
+                        tracing::warn!(error=%err, "internal keepalive failed to send keepalive");
+                        break;
+                    }
+
+                    match reader.next_line().await {
+                        Ok(Some(line)) => {
+                            tracing::debug!(?line, "internal keepalive response");
+                        }
+                        Ok(None) => {
+                            tracing::warn!("internal keepalive connection closed");
+                            break;
+                        }
+                        Err(err) => {
+                            tracing::warn!(error=%err, "internal keepalive failed to read response");
+                            break;
+                        }
+                    }
+
+                    sleep(Duration::from_millis(INTERNAL_KEEPALIVE_INTERVAL_MS)).await;
+                }
+            }
+            Err(err) => {
+                tracing::warn!(error=%err, socket=%socket_path.display(), "internal keepalive failed to connect");
+            }
+        }
+
+        sleep(Duration::from_secs(1)).await;
+    }
+}

rust/babblerd/src/profiling/estimator.rs

@@ -0,0 +1,124 @@
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct LatencyStats {
+    pub sent: u32,
+    pub received: u32,
+    pub loss_ratio: f64,
+    pub min: Duration,
+    pub avg: Duration,
+    pub max: Duration,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CapacitySample {
+    pub sent_packets: u32,
+    pub received_packets: u32,
+    pub received_bytes: u64,
+    pub span: Duration,
+}
+
+impl CapacitySample {
+    pub fn loss_ratio(self) -> f64 {
+        if self.sent_packets == 0 {
+            return 0.0;
+        }
+        let lost = self.sent_packets.saturating_sub(self.received_packets);
+        f64::from(lost) / f64::from(self.sent_packets)
+    }
+
+    pub fn mbps(self) -> Option<f64> {
+        capacity_mbps(self.received_bytes, self.span)
+    }
+}
+
+pub fn latency_stats(sent: u32, samples: &[Duration]) -> Option<LatencyStats> {
+    let received = u32::try_from(samples.len()).ok()?;
+    if sent == 0 || samples.is_empty() {
+        return None;
+    }
+
+    let mut min = samples.first().copied()?;
+    let mut max = min;
+    let mut total_nanos = 0_u128;
+
+    for sample in samples {
+        min = min.min(*sample);
+        max = max.max(*sample);
+        total_nanos = total_nanos.saturating_add(sample.as_nanos());
+    }
+
+    let avg_nanos = total_nanos / u128::from(received);
+    let avg = Duration::from_nanos(u64_saturating_from_u128(avg_nanos));
+    let lost = sent.saturating_sub(received);
+
+    Some(LatencyStats {
+        sent,
+        received,
+        loss_ratio: f64::from(lost) / f64::from(sent),
+        min,
+        avg,
+        max,
+    })
+}
+
+pub fn capacity_mbps(received_bytes: u64, span: Duration) -> Option<f64> {
+    let nanos = span.as_nanos();
+    if received_bytes == 0 || nanos == 0 {
+        return None;
+    }
+
+    let bits = received_bytes.saturating_mul(8);
+    Some((bits as f64) * 1_000.0 / (nanos as f64))
+}
+
+pub fn duration_nanos_u64(duration: Duration) -> u64 {
+    u64_saturating_from_u128(duration.as_nanos())
+}
+
+fn u64_saturating_from_u128(value: u128) -> u64 {
+    u64::try_from(value).unwrap_or(u64::MAX)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use super::{CapacitySample, capacity_mbps, latency_stats};
+
+    #[test]
+    fn latency_summary_reports_loss_and_bounds() {
+        let samples = [
+            Duration::from_millis(3),
+            Duration::from_millis(1),
+            Duration::from_millis(2),
+        ];
+
+        let Some(stats) = latency_stats(4, &samples) else {
+            panic!("expected latency stats");
+        };
+
+        assert_eq!(stats.sent, 4);
+        assert_eq!(stats.received, 3);
+        assert_eq!(stats.loss_ratio, 0.25);
+        assert_eq!(stats.min, Duration::from_millis(1));
+        assert_eq!(stats.avg, Duration::from_millis(2));
+        assert_eq!(stats.max, Duration::from_millis(3));
+    }
+
+    #[test]
+    fn capacity_summary_reports_mbps() {
+        let sample = CapacitySample {
+            sent_packets: 10,
+            received_packets: 10,
+            received_bytes: 125_000,
+            span: Duration::from_millis(1),
+        };
+
+        assert_eq!(sample.loss_ratio(), 0.0);
+        assert_eq!(
+            capacity_mbps(sample.received_bytes, sample.span),
+            Some(1000.0)
+        );
+    }
+}

rust/babblerd/src/profiling/mod.rs

@@ -0,0 +1,16 @@
+//! Link-local profiling support.
+//!
+//! This module is intentionally independent of the Babel control plane.  The
+//! standalone example uses it to measure one physical link directly; the daemon
+//! can later consume the same types and estimators when route scoring is wired
+//! in.
+
+pub mod estimator;
+pub mod pbprobe;
+pub mod protocol;
+pub mod socket;
+pub mod standalone;
+pub mod types;
+
+pub use estimator::{CapacitySample, LatencyStats, capacity_mbps, latency_stats};
+pub use types::{DEFAULT_PROFILE_PORT, LinkKey, ProbeConfig};

rust/babblerd/src/profiling/pbprobe/estimator.rs

@@ -0,0 +1,195 @@
+use std::time::Duration;
+
+use crate::config::{OUTER_IPV6_HEADER_BYTES, OUTER_UDP_HEADER_BYTES, PHYSICAL_LINK_MTU};
+
+pub const DEFAULT_PBPROBE_PORT: u16 = 41_902;
+pub const DEFAULT_SAMPLE_COUNT: u32 = 200;
+pub const DEFAULT_UTILIZATION: f64 = 0.01;
+pub const DEFAULT_DISPERSION_THRESHOLD_MS: u64 = 1;
+pub const DEFAULT_DISPERSION_THRESHOLD: Duration =
+    Duration::from_millis(DEFAULT_DISPERSION_THRESHOLD_MS);
+pub const DEFAULT_MAX_BULK_LEN: u32 = 10_000;
+pub const DEFAULT_RTS_TIMEOUT_MS: u64 = 750;
+pub const DEFAULT_RTS_TIMEOUT: Duration = Duration::from_millis(DEFAULT_RTS_TIMEOUT_MS);
+pub const DEFAULT_START_TIMEOUT_MS: u64 = 750;
+pub const DEFAULT_START_TIMEOUT: Duration = Duration::from_millis(DEFAULT_START_TIMEOUT_MS);
+pub const DEFAULT_CONTROL_RETRIES: u32 = 5;
+
+#[derive(Debug, Clone)]
+pub struct PbProbeConfig {
+    pub port: u16,
+    pub sample_count: u32,
+    pub utilization: f64,
+    pub dispersion_threshold: Duration,
+    pub initial_bulk_len: u32,
+    pub max_bulk_len: u32,
+    pub ip_packet_bytes: usize,
+    pub start_timeout: Duration,
+    pub rts_timeout: Duration,
+    pub control_retries: u32,
+}
+
+impl Default for PbProbeConfig {
+    fn default() -> Self {
+        Self {
+            port: DEFAULT_PBPROBE_PORT,
+            sample_count: DEFAULT_SAMPLE_COUNT,
+            utilization: DEFAULT_UTILIZATION,
+            dispersion_threshold: DEFAULT_DISPERSION_THRESHOLD,
+            initial_bulk_len: 1,
+            max_bulk_len: DEFAULT_MAX_BULK_LEN,
+            ip_packet_bytes: usize::from(PHYSICAL_LINK_MTU),
+            start_timeout: DEFAULT_START_TIMEOUT,
+            rts_timeout: DEFAULT_RTS_TIMEOUT,
+            control_retries: DEFAULT_CONTROL_RETRIES,
+        }
+    }
+}
+
+impl PbProbeConfig {
+    pub fn udp_payload_bytes(&self) -> usize {
+        let overhead = usize::from(OUTER_IPV6_HEADER_BYTES + OUTER_UDP_HEADER_BYTES);
+        self.ip_packet_bytes.saturating_sub(overhead)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AcceptedSample {
+    pub sample_id: u32,
+    pub bulk_len: u32,
+    pub delay_first: Duration,
+    pub delay_last: Duration,
+    pub dispersion: Duration,
+    pub server_issue_duration: Option<Duration>,
+}
+
+impl AcceptedSample {
+    pub fn delay_sum(self) -> Duration {
+        self.delay_first.saturating_add(self.delay_last)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct SelectedSample {
+    pub sample: AcceptedSample,
+    pub capacity_mbps: f64,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Estimate {
+    pub bulk_len: u32,
+    pub sample_count: u32,
+    pub attempts: u32,
+    pub lost_samples: u32,
+    pub ip_packet_bytes: usize,
+    pub selected: SelectedSample,
+    pub min_dispersion: Duration,
+    pub server_issue_samples: u32,
+    pub min_server_issue_duration: Option<Duration>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum EstimateOutcome {
+    Complete(Estimate),
+    IncreaseBulk {
+        previous_bulk_len: u32,
+        next_bulk_len: u32,
+        observed_dispersion: Duration,
+    },
+}
+
+pub fn select_capacity_sample(
+    samples: &[AcceptedSample],
+    ip_packet_bytes: usize,
+) -> Option<SelectedSample> {
+    let sample = samples
+        .iter()
+        .copied()
+        .min_by_key(|sample| sample.delay_sum())?;
+    let capacity_mbps = capacity_mbps(sample.bulk_len, ip_packet_bytes, sample.dispersion)?;
+    Some(SelectedSample {
+        sample,
+        capacity_mbps,
+    })
+}
+
+pub fn capacity_mbps(bulk_len: u32, ip_packet_bytes: usize, dispersion: Duration) -> Option<f64> {
+    let nanos = dispersion.as_nanos();
+    if bulk_len == 0 || ip_packet_bytes == 0 || nanos == 0 {
+        return None;
+    }
+
+    let bits = f64::from(bulk_len) * (ip_packet_bytes as f64) * 8.0;
+    Some(bits * 1_000.0 / (nanos as f64))
+}
+
+pub fn next_bulk_len(current: u32, max: u32) -> Option<u32> {
+    let next = current.checked_mul(10)?;
+    if next > max || next == current {
+        return None;
+    }
+    Some(next)
+}
+
+pub fn pacing_interval(dispersion: Duration, utilization: f64) -> Option<Duration> {
+    if dispersion.is_zero() || !utilization.is_finite() || utilization <= 0.0 {
+        return None;
+    }
+
+    Some(Duration::from_secs_f64(
+        (2.0 * dispersion.as_secs_f64()) / utilization,
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use super::{
+        AcceptedSample, capacity_mbps, next_bulk_len, pacing_interval, select_capacity_sample,
+    };
+
+    #[test]
+    fn capacity_uses_bulk_length_not_packet_count() {
+        let estimate = capacity_mbps(100, 1500, Duration::from_micros(1200));
+        assert_eq!(estimate, Some(1000.0));
+    }
+
+    #[test]
+    fn selector_uses_minimum_delay_sum() {
+        let samples = [
+            AcceptedSample {
+                sample_id: 1,
+                bulk_len: 10,
+                delay_first: Duration::from_millis(3),
+                delay_last: Duration::from_millis(4),
+                dispersion: Duration::from_micros(900),
+                server_issue_duration: None,
+            },
+            AcceptedSample {
+                sample_id: 2,
+                bulk_len: 10,
+                delay_first: Duration::from_millis(1),
+                delay_last: Duration::from_millis(2),
+                dispersion: Duration::from_micros(1200),
+                server_issue_duration: None,
+            },
+        ];
+
+        let selected = select_capacity_sample(&samples, 1500).expect("sample should be selected");
+        assert_eq!(selected.sample.sample_id, 2);
+        assert_eq!(selected.capacity_mbps, 100.0);
+    }
+
+    #[test]
+    fn bulk_growth_is_tenfold_and_capped() {
+        assert_eq!(next_bulk_len(1, 1000), Some(10));
+        assert_eq!(next_bulk_len(1000, 1000), None);
+    }
+
+    #[test]
+    fn pacing_follows_paper_formula() {
+        let interval = pacing_interval(Duration::from_millis(1), 0.01);
+        assert_eq!(interval, Some(Duration::from_millis(200)));
+    }
+}

rust/babblerd/src/profiling/pbprobe/mod.rs

@@ -0,0 +1,14 @@
+//! Paper-faithful PBProbe implementation.
+//!
+//! PBProbe is a CapProbe-derived capacity estimator that uses packet bulks
+//! instead of a single packet pair.  This module follows the paper algorithm
+//! rather than the old C implementation's process/control structure.
+
+pub mod estimator;
+pub mod protocol;
+pub mod standalone;
+
+pub use estimator::{
+    AcceptedSample, Estimate, EstimateOutcome, PbProbeConfig, SelectedSample, next_bulk_len,
+    pacing_interval, select_capacity_sample,
+};

rust/babblerd/src/profiling/pbprobe/protocol.rs

@@ -0,0 +1,317 @@
+use std::mem::size_of;
+use std::time::Duration;
+
+use thiserror::Error;
+use zerocopy::byteorder::{NetworkEndian, U16, U32, U64};
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+type U16Be = U16<NetworkEndian>;
+type U32Be = U32<NetworkEndian>;
+type U64Be = U64<NetworkEndian>;
+
+pub const HEADER_LEN: usize = size_of::<WireHeader>();
+pub const RESULT_BODY_LEN: usize = size_of::<WireResultBody>();
+pub const RESULT_PACKET_LEN: usize = HEADER_LEN + RESULT_BODY_LEN;
+
+const MAGIC: &[u8; 4] = b"BBPB";
+const VERSION: u8 = 1;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum PacketKind {
+    Start = 1,
+    StartAck = 2,
+    Rts = 3,
+    Bulk = 4,
+    Result = 5,
+    End = 6,
+    ErrorMessage = 7,
+}
+
+impl TryFrom<u8> for PacketKind {
+    type Error = ProtocolError;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            1 => Ok(Self::Start),
+            2 => Ok(Self::StartAck),
+            3 => Ok(Self::Rts),
+            4 => Ok(Self::Bulk),
+            5 => Ok(Self::Result),
+            6 => Ok(Self::End),
+            7 => Ok(Self::ErrorMessage),
+            other => Err(ProtocolError::UnknownKind(other)),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Header {
+    pub kind: PacketKind,
+    pub run_id: u64,
+    pub sample_id: u32,
+    pub seq: u32,
+    pub bulk_len: u32,
+    pub sample_count: u32,
+    pub ip_packet_bytes: u32,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ResultBody {
+    pub attempts: u32,
+    pub lost_samples: u32,
+    pub selected_sample_id: u32,
+    pub accepted_samples: u32,
+    pub delay_sum: Duration,
+    pub dispersion: Duration,
+    pub min_dispersion: Duration,
+    pub capacity_mbps: f64,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout)]
+struct WireHeader {
+    magic: [u8; 4],
+    version: u8,
+    kind: u8,
+    flags: U16Be,
+    run_id: U64Be,
+    sample_id: U32Be,
+    seq: U32Be,
+    bulk_len: U32Be,
+    sample_count: U32Be,
+    ip_packet_bytes: U32Be,
+    reserved: U32Be,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout)]
+struct WireResultBody {
+    attempts: U32Be,
+    lost_samples: U32Be,
+    selected_sample_id: U32Be,
+    accepted_samples: U32Be,
+    delay_sum_nanos: U64Be,
+    dispersion_nanos: U64Be,
+    min_dispersion_nanos: U64Be,
+    capacity_mbps_bits: U64Be,
+}
+
+#[derive(Debug, Error, Clone, PartialEq, Eq)]
+pub enum ProtocolError {
+    #[error("packet is too short")]
+    TooShort,
+    #[error("packet buffer is too small")]
+    BufferTooSmall,
+    #[error("bad PBProbe packet magic")]
+    BadMagic,
+    #[error("unsupported PBProbe protocol version {0}")]
+    BadVersion(u8),
+    #[error("unknown PBProbe packet kind {0}")]
+    UnknownKind(u8),
+}
+
+pub fn encode_header(dst: &mut [u8], header: Header) -> Result<usize, ProtocolError> {
+    encode_header_with_aux(dst, header, 0)
+}
+
+pub fn encode_header_with_aux(
+    dst: &mut [u8],
+    header: Header,
+    aux: u32,
+) -> Result<usize, ProtocolError> {
+    write_bytes(dst, WireHeader::from_header(header, aux).as_bytes())
+}
+
+pub fn decode_header(src: &[u8]) -> Result<Header, ProtocolError> {
+    decode_header_with_aux(src).map(|(header, _aux)| header)
+}
+
+pub fn decode_header_with_aux(src: &[u8]) -> Result<(Header, u32), ProtocolError> {
+    let (wire, _) = WireHeader::read_from_prefix(src).map_err(|_| ProtocolError::TooShort)?;
+    wire.decode()
+}
+
+pub fn encode_result(
+    dst: &mut [u8],
+    header: Header,
+    body: ResultBody,
+) -> Result<usize, ProtocolError> {
+    if dst.len() < RESULT_PACKET_LEN {
+        return Err(ProtocolError::BufferTooSmall);
+    }
+
+    let cursor = encode_header(dst, header)?;
+    write_bytes(&mut dst[cursor..], WireResultBody::from(body).as_bytes())?;
+    Ok(RESULT_PACKET_LEN)
+}
+
+pub fn decode_result_body(src: &[u8]) -> Result<ResultBody, ProtocolError> {
+    let body_src = src.get(HEADER_LEN..).ok_or(ProtocolError::TooShort)?;
+    let (wire, _) =
+        WireResultBody::read_from_prefix(body_src).map_err(|_| ProtocolError::TooShort)?;
+    Ok(ResultBody::from(wire))
+}
+
+pub fn duration_nanos(duration: Duration) -> u64 {
+    u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
+}
+
+impl WireHeader {
+    fn from_header(header: Header, aux: u32) -> Self {
+        Self {
+            magic: *MAGIC,
+            version: VERSION,
+            kind: header.kind as u8,
+            flags: U16Be::ZERO,
+            run_id: U64Be::new(header.run_id),
+            sample_id: U32Be::new(header.sample_id),
+            seq: U32Be::new(header.seq),
+            bulk_len: U32Be::new(header.bulk_len),
+            sample_count: U32Be::new(header.sample_count),
+            ip_packet_bytes: U32Be::new(header.ip_packet_bytes),
+            reserved: U32Be::new(aux),
+        }
+    }
+
+    fn decode(self) -> Result<(Header, u32), ProtocolError> {
+        if self.magic != *MAGIC {
+            return Err(ProtocolError::BadMagic);
+        }
+        if self.version != VERSION {
+            return Err(ProtocolError::BadVersion(self.version));
+        }
+
+        Ok((
+            Header {
+                kind: PacketKind::try_from(self.kind)?,
+                run_id: self.run_id.get(),
+                sample_id: self.sample_id.get(),
+                seq: self.seq.get(),
+                bulk_len: self.bulk_len.get(),
+                sample_count: self.sample_count.get(),
+                ip_packet_bytes: self.ip_packet_bytes.get(),
+            },
+            self.reserved.get(),
+        ))
+    }
+}
+
+impl From<ResultBody> for WireResultBody {
+    fn from(body: ResultBody) -> Self {
+        Self {
+            attempts: U32Be::new(body.attempts),
+            lost_samples: U32Be::new(body.lost_samples),
+            selected_sample_id: U32Be::new(body.selected_sample_id),
+            accepted_samples: U32Be::new(body.accepted_samples),
+            delay_sum_nanos: U64Be::new(duration_nanos(body.delay_sum)),
+            dispersion_nanos: U64Be::new(duration_nanos(body.dispersion)),
+            min_dispersion_nanos: U64Be::new(duration_nanos(body.min_dispersion)),
+            capacity_mbps_bits: U64Be::new(body.capacity_mbps.to_bits()),
+        }
+    }
+}
+
+impl From<WireResultBody> for ResultBody {
+    fn from(wire: WireResultBody) -> Self {
+        Self {
+            attempts: wire.attempts.get(),
+            lost_samples: wire.lost_samples.get(),
+            selected_sample_id: wire.selected_sample_id.get(),
+            accepted_samples: wire.accepted_samples.get(),
+            delay_sum: Duration::from_nanos(wire.delay_sum_nanos.get()),
+            dispersion: Duration::from_nanos(wire.dispersion_nanos.get()),
+            min_dispersion: Duration::from_nanos(wire.min_dispersion_nanos.get()),
+            capacity_mbps: f64::from_bits(wire.capacity_mbps_bits.get()),
+        }
+    }
+}
+
+fn write_bytes(dst: &mut [u8], src: &[u8]) -> Result<usize, ProtocolError> {
+    let Some(slot) = dst.get_mut(..src.len()) else {
+        return Err(ProtocolError::BufferTooSmall);
+    };
+    slot.copy_from_slice(src);
+    Ok(src.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use super::{
+        HEADER_LEN, Header, PacketKind, RESULT_PACKET_LEN, ResultBody, decode_header,
+        decode_header_with_aux, decode_result_body, encode_header, encode_header_with_aux,
+        encode_result,
+    };
+
+    #[test]
+    fn header_layout_is_stable() {
+        assert_eq!(HEADER_LEN, 40);
+    }
+
+    #[test]
+    fn header_round_trips() {
+        let header = Header {
+            kind: PacketKind::Bulk,
+            run_id: 7,
+            sample_id: 11,
+            seq: 3,
+            bulk_len: 100,
+            sample_count: 200,
+            ip_packet_bytes: 1500,
+        };
+        let mut buf = [0_u8; HEADER_LEN];
+
+        assert_eq!(encode_header(&mut buf, header), Ok(HEADER_LEN));
+        assert_eq!(decode_header(&buf), Ok(header));
+    }
+
+    #[test]
+    fn header_aux_round_trips() {
+        let header = Header {
+            kind: PacketKind::Bulk,
+            run_id: 7,
+            sample_id: 11,
+            seq: 100,
+            bulk_len: 100,
+            sample_count: 200,
+            ip_packet_bytes: 1500,
+        };
+        let mut buf = [0_u8; HEADER_LEN];
+
+        assert_eq!(
+            encode_header_with_aux(&mut buf, header, 12_345),
+            Ok(HEADER_LEN)
+        );
+        assert_eq!(decode_header_with_aux(&buf), Ok((header, 12_345)));
+    }
+
+    #[test]
+    fn result_round_trips() {
+        let header = Header {
+            kind: PacketKind::Result,
+            run_id: 9,
+            sample_id: 0,
+            seq: 0,
+            bulk_len: 100,
+            sample_count: 200,
+            ip_packet_bytes: 1500,
+        };
+        let body = ResultBody {
+            attempts: 210,
+            lost_samples: 10,
+            selected_sample_id: 42,
+            accepted_samples: 200,
+            delay_sum: Duration::from_micros(123),
+            dispersion: Duration::from_micros(1200),
+            min_dispersion: Duration::from_micros(1100),
+            capacity_mbps: 1000.25,
+        };
+        let mut buf = [0_u8; RESULT_PACKET_LEN];
+
+        assert_eq!(encode_result(&mut buf, header, body), Ok(RESULT_PACKET_LEN));
+        assert_eq!(decode_header(&buf), Ok(header));
+        assert_eq!(decode_result_body(&buf), Ok(body));
+    }
+}

rust/babblerd/src/profiling/protocol.rs

@@ -0,0 +1,228 @@
+use std::mem::size_of;
+
+use thiserror::Error;
+use zerocopy::byteorder::{NetworkEndian, U16, U32, U64};
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+type U16Be = U16<NetworkEndian>;
+type U32Be = U32<NetworkEndian>;
+type U64Be = U64<NetworkEndian>;
+
+pub const HEADER_LEN: usize = size_of::<WireHeader>();
+pub const SUMMARY_BODY_LEN: usize = size_of::<WireSummaryBody>();
+pub const SUMMARY_PACKET_LEN: usize = HEADER_LEN + SUMMARY_BODY_LEN;
+
+const MAGIC: &[u8; 4] = b"BBLP";
+const VERSION: u8 = 1;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum PacketKind {
+    EchoRequest = 1,
+    EchoReply = 2,
+    Train = 3,
+    SummaryRequest = 4,
+    SummaryReply = 5,
+}
+
+impl TryFrom<u8> for PacketKind {
+    type Error = ProtocolError;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            1 => Ok(Self::EchoRequest),
+            2 => Ok(Self::EchoReply),
+            3 => Ok(Self::Train),
+            4 => Ok(Self::SummaryRequest),
+            5 => Ok(Self::SummaryReply),
+            other => Err(ProtocolError::UnknownKind(other)),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Header {
+    pub kind: PacketKind,
+    pub run_id: u64,
+    pub seq: u32,
+    pub count: u32,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct SummaryBody {
+    pub received_packets: u32,
+    pub received_bytes: u64,
+    pub span_nanos: u64,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout)]
+struct WireHeader {
+    magic: [u8; 4],
+    version: u8,
+    kind: u8,
+    flags: U16Be,
+    run_id: U64Be,
+    seq: U32Be,
+    count: U32Be,
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy, FromBytes, Immutable, IntoBytes, KnownLayout)]
+struct WireSummaryBody {
+    received_packets: U32Be,
+    received_bytes: U64Be,
+    span_nanos: U64Be,
+}
+
+#[derive(Debug, Error, Clone, PartialEq, Eq)]
+pub enum ProtocolError {
+    #[error("packet is too short")]
+    TooShort,
+    #[error("packet buffer is too small")]
+    BufferTooSmall,
+    #[error("bad profiling packet magic")]
+    BadMagic,
+    #[error("unsupported profiling protocol version {0}")]
+    BadVersion(u8),
+    #[error("unknown profiling packet kind {0}")]
+    UnknownKind(u8),
+}
+
+pub fn encode_header(dst: &mut [u8], header: Header) -> Result<usize, ProtocolError> {
+    write_bytes(dst, WireHeader::from_header(header).as_bytes())
+}
+
+pub fn decode_header(src: &[u8]) -> Result<Header, ProtocolError> {
+    let (wire, _) = WireHeader::read_from_prefix(src).map_err(|_| ProtocolError::TooShort)?;
+    wire.decode()
+}
+
+pub fn encode_summary(
+    dst: &mut [u8],
+    header: Header,
+    body: SummaryBody,
+) -> Result<usize, ProtocolError> {
+    if dst.len() < SUMMARY_PACKET_LEN {
+        return Err(ProtocolError::BufferTooSmall);
+    }
+
+    let cursor = encode_header(dst, header)?;
+    write_bytes(&mut dst[cursor..], WireSummaryBody::from(body).as_bytes())?;
+    Ok(SUMMARY_PACKET_LEN)
+}
+
+pub fn decode_summary_body(src: &[u8]) -> Result<SummaryBody, ProtocolError> {
+    let body_src = src.get(HEADER_LEN..).ok_or(ProtocolError::TooShort)?;
+    let (wire, _) =
+        WireSummaryBody::read_from_prefix(body_src).map_err(|_| ProtocolError::TooShort)?;
+    Ok(SummaryBody::from(wire))
+}
+
+impl WireHeader {
+    fn from_header(header: Header) -> Self {
+        Self {
+            magic: *MAGIC,
+            version: VERSION,
+            kind: header.kind as u8,
+            flags: U16Be::ZERO,
+            run_id: U64Be::new(header.run_id),
+            seq: U32Be::new(header.seq),
+            count: U32Be::new(header.count),
+        }
+    }
+
+    fn decode(self) -> Result<Header, ProtocolError> {
+        if self.magic != *MAGIC {
+            return Err(ProtocolError::BadMagic);
+        }
+        if self.version != VERSION {
+            return Err(ProtocolError::BadVersion(self.version));
+        }
+
+        Ok(Header {
+            kind: PacketKind::try_from(self.kind)?,
+            run_id: self.run_id.get(),
+            seq: self.seq.get(),
+            count: self.count.get(),
+        })
+    }
+}
+
+impl From<SummaryBody> for WireSummaryBody {
+    fn from(body: SummaryBody) -> Self {
+        Self {
+            received_packets: U32Be::new(body.received_packets),
+            received_bytes: U64Be::new(body.received_bytes),
+            span_nanos: U64Be::new(body.span_nanos),
+        }
+    }
+}
+
+impl From<WireSummaryBody> for SummaryBody {
+    fn from(wire: WireSummaryBody) -> Self {
+        Self {
+            received_packets: wire.received_packets.get(),
+            received_bytes: wire.received_bytes.get(),
+            span_nanos: wire.span_nanos.get(),
+        }
+    }
+}
+
+fn write_bytes(dst: &mut [u8], src: &[u8]) -> Result<usize, ProtocolError> {
+    let Some(slot) = dst.get_mut(..src.len()) else {
+        return Err(ProtocolError::BufferTooSmall);
+    };
+    slot.copy_from_slice(src);
+    Ok(src.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        HEADER_LEN, Header, PacketKind, SUMMARY_PACKET_LEN, SummaryBody, decode_header,
+        decode_summary_body, encode_header, encode_summary,
+    };
+
+    #[test]
+    fn layouts_are_stable() {
+        assert_eq!(HEADER_LEN, 24);
+        assert_eq!(SUMMARY_PACKET_LEN, 44);
+    }
+
+    #[test]
+    fn header_round_trips() {
+        let header = Header {
+            kind: PacketKind::Train,
+            run_id: 42,
+            seq: 7,
+            count: 64,
+        };
+        let mut buf = [0_u8; HEADER_LEN];
+
+        let encoded = encode_header(&mut buf, header);
+        assert_eq!(encoded, Ok(HEADER_LEN));
+        assert_eq!(decode_header(&buf), Ok(header));
+    }
+
+    #[test]
+    fn summary_round_trips() {
+        let header = Header {
+            kind: PacketKind::SummaryReply,
+            run_id: 99,
+            seq: 0,
+            count: 64,
+        };
+        let body = SummaryBody {
+            received_packets: 63,
+            received_bytes: 91_476,
+            span_nanos: 725_000,
+        };
+        let mut buf = [0_u8; SUMMARY_PACKET_LEN];
+
+        let encoded = encode_summary(&mut buf, header, body);
+        assert_eq!(encoded, Ok(SUMMARY_PACKET_LEN));
+        assert_eq!(decode_header(&buf), Ok(header));
+        assert_eq!(decode_summary_body(&buf), Ok(body));
+    }
+}

rust/babblerd/src/profiling/socket.rs

@@ -0,0 +1,63 @@
+use std::io;
+use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6, UdpSocket};
+use std::num::NonZeroU32;
+use std::time::Duration;
+
+use nix::net::if_::if_nametoindex;
+use socket2::{Domain, Protocol, Socket, Type};
+
+const PROFILE_SOCKET_BUFFER_BYTES: usize = 4 * 1024 * 1024;
+
+pub fn open_link_local_udp(
+    ifname: &str,
+    port: u16,
+    read_timeout: Option<Duration>,
+) -> io::Result<(UdpSocket, u32)> {
+    let ifindex = if_nametoindex(ifname).map_err(io::Error::from)?;
+    let Some(nonzero_ifindex) = NonZeroU32::new(ifindex) else {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!("invalid ifindex for {ifname}"),
+        ));
+    };
+
+    let socket = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))?;
+    socket.set_reuse_address(true)?;
+    socket.set_reuse_port(true)?;
+    socket.set_only_v6(true)?;
+    socket.set_recv_buffer_size(PROFILE_SOCKET_BUFFER_BYTES)?;
+    socket.set_send_buffer_size(PROFILE_SOCKET_BUFFER_BYTES)?;
+
+    #[cfg(target_os = "linux")]
+    socket.bind_device(Some(ifname.as_bytes()))?;
+    socket.bind_device_by_index_v6(Some(nonzero_ifindex))?;
+    socket.bind(&SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, port, 0, 0).into())?;
+
+    let udp: UdpSocket = socket.into();
+    udp.set_read_timeout(read_timeout)?;
+    udp.set_write_timeout(read_timeout)?;
+    Ok((udp, ifindex))
+}
+
+pub fn scoped_peer_addr(peer: Ipv6Addr, port: u16, ifindex: u32) -> SocketAddr {
+    SocketAddr::V6(SocketAddrV6::new(peer, port, 0, ifindex))
+}
+
+pub fn with_default_scope(addr: SocketAddr, ifindex: u32) -> SocketAddr {
+    match addr {
+        SocketAddr::V6(v6) if v6.ip().is_unicast_link_local() && v6.scope_id() == 0 => {
+            SocketAddr::V6(SocketAddrV6::new(
+                *v6.ip(),
+                v6.port(),
+                v6.flowinfo(),
+                ifindex,
+            ))
+        }
+        other => other,
+    }
+}
+
+pub fn parse_link_local_addr(raw: &str) -> Result<Ipv6Addr, std::net::AddrParseError> {
+    let addr = raw.split_once('%').map_or(raw, |(addr, _scope)| addr);
+    addr.parse()
+}

rust/babblerd/src/profiling/standalone.rs

@@ -0,0 +1,722 @@
+use std::collections::HashMap;
+use std::ffi::OsString;
+use std::io::{self, ErrorKind};
+use std::net::{Ipv6Addr, SocketAddr, UdpSocket};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use clap::{Args, Parser, Subcommand};
+use color_eyre::eyre::{Result, WrapErr, eyre};
+
+use super::estimator::{CapacitySample, duration_nanos_u64, latency_stats};
+use super::protocol::{
+    HEADER_LEN, Header, PacketKind, SUMMARY_PACKET_LEN, SummaryBody, decode_header,
+    decode_summary_body, encode_header, encode_summary,
+};
+use super::socket::{
+    open_link_local_udp, parse_link_local_addr, scoped_peer_addr, with_default_scope,
+};
+use super::types::{
+    DEFAULT_CAPACITY_ROUNDS, DEFAULT_ECHO_COUNT, DEFAULT_ECHO_INTERVAL_MS, DEFAULT_ECHO_TIMEOUT_MS,
+    DEFAULT_PROFILE_PORT, DEFAULT_TRAIN_INTERVAL_MS, DEFAULT_TRAIN_PACKETS,
+    DEFAULT_TRAIN_SETTLE_MS, ProbeConfig,
+};
+
+const MAX_UDP_PACKET_BYTES: usize = 65_535;
+const REFLECT_RECV_TIMEOUT: Duration = Duration::from_secs(1);
+const STALE_TRAIN_AFTER: Duration = Duration::from_secs(60);
+const SUMMARY_REQUEST_ATTEMPTS: u32 = 3;
+
+pub fn run_from_env() -> Result<()> {
+    run_cli(Cli::parse())
+}
+
+pub fn run<I, S>(args: I) -> Result<()>
+where
+    I: IntoIterator<Item = S>,
+    S: Into<OsString> + Clone,
+{
+    run_cli(Cli::try_parse_from(args)?)
+}
+
+fn run_cli(cli: Cli) -> Result<()> {
+    match cli.command {
+        Command::Probe(args) => run_probe(args.into_probe_options()),
+        Command::Reflect(options) => run_reflect(options),
+    }
+}
+
+#[derive(Debug, Parser)]
+#[command(name = "link_profile")]
+#[command(about = "Run standalone link-local latency and packet-train probes")]
+#[command(arg_required_else_help = true)]
+struct Cli {
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Debug, Subcommand)]
+enum Command {
+    Probe(ProbeArgs),
+    Reflect(ReflectOptions),
+}
+
+#[derive(Debug, Clone)]
+struct ProbeOptions {
+    ifname: String,
+    peer: Ipv6Addr,
+    config: ProbeConfig,
+}
+
+#[derive(Debug, Args, Clone)]
+struct ReflectOptions {
+    #[arg(long)]
+    ifname: String,
+
+    #[arg(long, default_value_t = DEFAULT_PROFILE_PORT)]
+    port: u16,
+}
+
+#[derive(Debug, Args, Clone)]
+struct ProbeArgs {
+    #[arg(long)]
+    ifname: String,
+
+    #[arg(long, value_parser = parse_link_local_addr_arg)]
+    peer: Ipv6Addr,
+
+    #[arg(long, default_value_t = DEFAULT_PROFILE_PORT)]
+    port: u16,
+
+    #[arg(long = "echo-count", default_value_t = DEFAULT_ECHO_COUNT)]
+    echo_count: u32,
+
+    #[arg(long = "echo-interval-ms", default_value_t = DEFAULT_ECHO_INTERVAL_MS)]
+    echo_interval_ms: u64,
+
+    #[arg(long = "timeout-ms", default_value_t = DEFAULT_ECHO_TIMEOUT_MS)]
+    timeout_ms: u64,
+
+    #[arg(long = "capacity-rounds", default_value_t = DEFAULT_CAPACITY_ROUNDS)]
+    capacity_rounds: u32,
+
+    #[arg(long = "train-packets", default_value_t = DEFAULT_TRAIN_PACKETS)]
+    train_packets: u32,
+
+    #[arg(long = "payload-bytes", default_value_t = usize::from(crate::config::TUN_MTU))]
+    payload_bytes: usize,
+
+    #[arg(long = "train-interval-ms", default_value_t = DEFAULT_TRAIN_INTERVAL_MS)]
+    train_interval_ms: u64,
+
+    #[arg(long = "settle-ms", default_value_t = DEFAULT_TRAIN_SETTLE_MS)]
+    settle_ms: u64,
+}
+
+impl ProbeArgs {
+    fn into_probe_options(self) -> ProbeOptions {
+        ProbeOptions {
+            ifname: self.ifname,
+            peer: self.peer,
+            config: ProbeConfig {
+                port: self.port,
+                echo_count: self.echo_count,
+                echo_interval: Duration::from_millis(self.echo_interval_ms),
+                echo_timeout: Duration::from_millis(self.timeout_ms),
+                capacity_rounds: self.capacity_rounds,
+                train_packets: self.train_packets,
+                train_payload_bytes: self.payload_bytes,
+                train_interval: Duration::from_millis(self.train_interval_ms),
+                train_settle: Duration::from_millis(self.settle_ms),
+            },
+        }
+    }
+}
+
+#[derive(Debug)]
+struct TrainAccumulator {
+    received_packets: u32,
+    received_bytes: u64,
+    first_rx: Option<Instant>,
+    last_rx: Option<Instant>,
+    last_update: Instant,
+    seen: Vec<bool>,
+}
+
+impl TrainAccumulator {
+    fn new(expected_packets: u32, now: Instant) -> Self {
+        Self {
+            received_packets: 0,
+            received_bytes: 0,
+            first_rx: None,
+            last_rx: None,
+            last_update: now,
+            seen: vec![false; usize::try_from(expected_packets).unwrap_or(0)],
+        }
+    }
+
+    fn record(&mut self, seq: u32, packet_len: usize, now: Instant) {
+        self.last_update = now;
+        if !mark_seen(&mut self.seen, seq) {
+            return;
+        }
+
+        self.received_packets = self.received_packets.saturating_add(1);
+        self.received_bytes = self
+            .received_bytes
+            .saturating_add(u64::try_from(packet_len).unwrap_or(u64::MAX));
+        if self.first_rx.is_none() {
+            self.first_rx = Some(now);
+        }
+        self.last_rx = Some(now);
+    }
+
+    fn summary(&self) -> SummaryBody {
+        let span_nanos = match (self.first_rx, self.last_rx) {
+            (Some(first), Some(last)) => duration_nanos_u64(last.saturating_duration_since(first)),
+            _ => 0,
+        };
+
+        SummaryBody {
+            received_packets: self.received_packets,
+            received_bytes: self.received_bytes,
+            span_nanos,
+        }
+    }
+}
+
+fn mark_seen(seen: &mut [bool], seq: u32) -> bool {
+    let Ok(index) = usize::try_from(seq) else {
+        return false;
+    };
+    let Some(slot) = seen.get_mut(index) else {
+        return false;
+    };
+    if *slot {
+        return false;
+    }
+    *slot = true;
+    true
+}
+
+fn run_reflect(options: ReflectOptions) -> Result<()> {
+    let (socket, ifindex) =
+        open_link_local_udp(&options.ifname, options.port, Some(REFLECT_RECV_TIMEOUT))
+            .wrap_err_with(|| format!("opening profiling reflector on {}", options.ifname))?;
+    let local_addr = socket
+        .local_addr()
+        .wrap_err("reading reflector local address")?;
+
+    println!(
+        "reflecting profiling probes on {} ifindex={} local={}",
+        options.ifname, ifindex, local_addr
+    );
+
+    let mut buf = vec![0_u8; MAX_UDP_PACKET_BYTES];
+    let mut trains = HashMap::<u64, TrainAccumulator>::new();
+    let mut last_cleanup = Instant::now();
+
+    loop {
+        cleanup_stale_trains(&mut trains, &mut last_cleanup);
+
+        let (packet_len, from) = match socket.recv_from(&mut buf) {
+            Ok(received) => received,
+            Err(err) if is_timeout(&err) || err.kind() == ErrorKind::Interrupted => continue,
+            Err(err) => return Err(err).wrap_err("receiving profiling packet"),
+        };
+
+        let Some(packet) = buf.get(..packet_len) else {
+            continue;
+        };
+        let Ok(header) = decode_header(packet) else {
+            continue;
+        };
+
+        match header.kind {
+            PacketKind::EchoRequest => {
+                send_echo_reply(&socket, ifindex, from, header).wrap_err("sending echo reply")?;
+            }
+            PacketKind::Train => {
+                let now = Instant::now();
+                trains
+                    .entry(header.run_id)
+                    .or_insert_with(|| TrainAccumulator::new(header.count, now))
+                    .record(header.seq, packet_len, now);
+            }
+            PacketKind::SummaryRequest => {
+                send_summary_reply(&socket, ifindex, from, header, &mut trains)
+                    .wrap_err("sending train summary")?;
+            }
+            PacketKind::EchoReply | PacketKind::SummaryReply => {}
+        }
+    }
+}
+
+fn run_probe(options: ProbeOptions) -> Result<()> {
+    if options.config.train_payload_bytes < HEADER_LEN {
+        return Err(eyre!(
+            "train payload must be at least {HEADER_LEN} bytes, got {}",
+            options.config.train_payload_bytes
+        ));
+    }
+    if options.config.train_packets == 0 {
+        return Err(eyre!("train packet count must be non-zero"));
+    }
+
+    let (socket, ifindex) =
+        open_link_local_udp(&options.ifname, 0, Some(options.config.echo_timeout))
+            .wrap_err_with(|| format!("opening profiling probe socket on {}", options.ifname))?;
+    let peer = scoped_peer_addr(options.peer, options.config.port, ifindex);
+    let local_addr = socket
+        .local_addr()
+        .wrap_err("reading probe local address")?;
+    let base_run_id = make_base_run_id();
+
+    println!(
+        "probing {} via {} ifindex={} local={} peer_port={}",
+        options.peer, options.ifname, ifindex, local_addr, options.config.port
+    );
+    println!(
+        "capacity probe: rounds={} train_packets={} payload_bytes={} interval_ms={}",
+        options.config.capacity_rounds,
+        options.config.train_packets,
+        options.config.train_payload_bytes,
+        options.config.train_interval.as_millis()
+    );
+
+    let latency_samples = run_echo_probes(&socket, peer, base_run_id, &options.config)
+        .wrap_err("running latency probes")?;
+    print_latency_summary(options.config.echo_count, &latency_samples);
+
+    let capacity_samples = run_capacity_probes(&socket, peer, base_run_id, &options.config)
+        .wrap_err("running capacity probes")?;
+    print_capacity_summary(&capacity_samples);
+
+    Ok(())
+}
+
+fn send_echo_reply(
+    socket: &UdpSocket,
+    ifindex: u32,
+    from: SocketAddr,
+    request: Header,
+) -> Result<()> {
+    let reply = Header {
+        kind: PacketKind::EchoReply,
+        run_id: request.run_id,
+        seq: request.seq,
+        count: request.count,
+    };
+    let mut out = [0_u8; HEADER_LEN];
+    encode_header(&mut out, reply)?;
+    send_datagram(socket, &out, with_default_scope(from, ifindex))
+}
+
+fn send_summary_reply(
+    socket: &UdpSocket,
+    ifindex: u32,
+    from: SocketAddr,
+    request: Header,
+    trains: &mut HashMap<u64, TrainAccumulator>,
+) -> Result<()> {
+    let body = trains
+        .get(&request.run_id)
+        .map_or_else(empty_summary, TrainAccumulator::summary);
+    let reply = Header {
+        kind: PacketKind::SummaryReply,
+        run_id: request.run_id,
+        seq: 0,
+        count: request.count,
+    };
+    let mut out = [0_u8; SUMMARY_PACKET_LEN];
+    let len = encode_summary(&mut out, reply, body)?;
+    send_datagram(
+        socket,
+        out.get(..len).unwrap_or(&out),
+        with_default_scope(from, ifindex),
+    )?;
+    trains.remove(&request.run_id);
+    Ok(())
+}
+
+fn run_echo_probes(
+    socket: &UdpSocket,
+    peer: SocketAddr,
+    base_run_id: u64,
+    config: &ProbeConfig,
+) -> Result<Vec<Duration>> {
+    let mut samples = Vec::new();
+    let run_id = base_run_id ^ 0xe0c0_u64;
+
+    for seq in 0..config.echo_count {
+        let header = Header {
+            kind: PacketKind::EchoRequest,
+            run_id,
+            seq,
+            count: config.echo_count,
+        };
+        let mut out = [0_u8; HEADER_LEN];
+        encode_header(&mut out, header)?;
+
+        let start = Instant::now();
+        send_datagram(socket, &out, peer)?;
+        match receive_echo_reply(socket, run_id, seq, start, config.echo_timeout)? {
+            Some(sample) => {
+                println!("echo {:>3}: {}", seq + 1, format_duration(sample));
+                samples.push(sample);
+            }
+            None => {
+                println!("echo {:>3}: timeout", seq + 1);
+            }
+        }
+
+        thread::sleep(config.echo_interval);
+    }
+
+    Ok(samples)
+}
+
+fn run_capacity_probes(
+    socket: &UdpSocket,
+    peer: SocketAddr,
+    base_run_id: u64,
+    config: &ProbeConfig,
+) -> Result<Vec<CapacitySample>> {
+    let mut samples = Vec::new();
+    let mut train = vec![0_u8; config.train_payload_bytes];
+
+    for round in 0..config.capacity_rounds {
+        let run_id = base_run_id ^ (0xc0_ffee_u64.wrapping_add(u64::from(round)));
+        let sender_start = Instant::now();
+
+        for seq in 0..config.train_packets {
+            let header = Header {
+                kind: PacketKind::Train,
+                run_id,
+                seq,
+                count: config.train_packets,
+            };
+            encode_header(&mut train, header)?;
+            send_datagram(socket, &train, peer)?;
+        }
+
+        let sender_span = sender_start.elapsed();
+        thread::sleep(config.train_settle);
+
+        let summary = request_summary(socket, peer, run_id, config)?;
+        match summary {
+            Some(body) => {
+                let sample = CapacitySample {
+                    sent_packets: config.train_packets,
+                    received_packets: body.received_packets,
+                    received_bytes: body.received_bytes,
+                    span: Duration::from_nanos(body.span_nanos),
+                };
+                print_capacity_round(round + 1, sample, sender_span);
+                samples.push(sample);
+            }
+            None => {
+                println!(
+                    "capacity {:>3}: summary timeout after sender_burst={}",
+                    round + 1,
+                    format_duration(sender_span)
+                );
+            }
+        }
+
+        thread::sleep(config.train_interval);
+    }
+
+    Ok(samples)
+}
+
+fn request_summary(
+    socket: &UdpSocket,
+    peer: SocketAddr,
+    run_id: u64,
+    config: &ProbeConfig,
+) -> Result<Option<SummaryBody>> {
+    let request = Header {
+        kind: PacketKind::SummaryRequest,
+        run_id,
+        seq: 0,
+        count: config.train_packets,
+    };
+    let mut out = [0_u8; HEADER_LEN];
+    encode_header(&mut out, request)?;
+
+    let attempt_timeout = div_duration(config.echo_timeout, SUMMARY_REQUEST_ATTEMPTS);
+    for _attempt in 0..SUMMARY_REQUEST_ATTEMPTS {
+        send_datagram(socket, &out, peer)?;
+        let deadline = Instant::now() + attempt_timeout;
+        if let Some(summary) = receive_summary_reply(socket, run_id, deadline)? {
+            return Ok(Some(summary));
+        }
+    }
+
+    Ok(None)
+}
+
+fn receive_echo_reply(
+    socket: &UdpSocket,
+    run_id: u64,
+    seq: u32,
+    start: Instant,
+    timeout: Duration,
+) -> Result<Option<Duration>> {
+    let deadline = start + timeout;
+    let mut buf = vec![0_u8; MAX_UDP_PACKET_BYTES];
+
+    loop {
+        if !set_timeout_until(socket, deadline)? {
+            return Ok(None);
+        }
+
+        let (packet_len, _from) = match socket.recv_from(&mut buf) {
+            Ok(received) => received,
+            Err(err) if is_timeout(&err) => return Ok(None),
+            Err(err) if err.kind() == ErrorKind::Interrupted => continue,
+            Err(err) => return Err(err).wrap_err("receiving echo reply"),
+        };
+        let Some(packet) = buf.get(..packet_len) else {
+            continue;
+        };
+        let Ok(header) = decode_header(packet) else {
+            continue;
+        };
+        if header.kind == PacketKind::EchoReply && header.run_id == run_id && header.seq == seq {
+            return Ok(Some(start.elapsed()));
+        }
+    }
+}
+
+fn receive_summary_reply(
+    socket: &UdpSocket,
+    run_id: u64,
+    deadline: Instant,
+) -> Result<Option<SummaryBody>> {
+    let mut buf = vec![0_u8; MAX_UDP_PACKET_BYTES];
+
+    loop {
+        if !set_timeout_until(socket, deadline)? {
+            return Ok(None);
+        }
+
+        let (packet_len, _from) = match socket.recv_from(&mut buf) {
+            Ok(received) => received,
+            Err(err) if is_timeout(&err) => return Ok(None),
+            Err(err) if err.kind() == ErrorKind::Interrupted => continue,
+            Err(err) => return Err(err).wrap_err("receiving train summary"),
+        };
+        let Some(packet) = buf.get(..packet_len) else {
+            continue;
+        };
+        let Ok(header) = decode_header(packet) else {
+            continue;
+        };
+        if header.kind == PacketKind::SummaryReply && header.run_id == run_id {
+            return Ok(Some(decode_summary_body(packet)?));
+        }
+    }
+}
+
+fn send_datagram(socket: &UdpSocket, buf: &[u8], target: SocketAddr) -> Result<()> {
+    let sent = socket
+        .send_to(buf, target)
+        .wrap_err_with(|| format!("sending profiling packet to {target}"))?;
+    if sent != buf.len() {
+        return Err(eyre!(
+            "short UDP send to {target}: sent {sent} of {} bytes",
+            buf.len()
+        ));
+    }
+    Ok(())
+}
+
+fn set_timeout_until(socket: &UdpSocket, deadline: Instant) -> Result<bool> {
+    let now = Instant::now();
+    if now >= deadline {
+        return Ok(false);
+    }
+    socket
+        .set_read_timeout(Some(deadline.saturating_duration_since(now)))
+        .wrap_err("setting profiling socket read timeout")?;
+    Ok(true)
+}
+
+fn cleanup_stale_trains(trains: &mut HashMap<u64, TrainAccumulator>, last_cleanup: &mut Instant) {
+    if last_cleanup.elapsed() < Duration::from_secs(5) {
+        return;
+    }
+
+    trains.retain(|_run_id, train| train.last_update.elapsed() < STALE_TRAIN_AFTER);
+    *last_cleanup = Instant::now();
+}
+
+fn empty_summary() -> SummaryBody {
+    SummaryBody {
+        received_packets: 0,
+        received_bytes: 0,
+        span_nanos: 0,
+    }
+}
+
+fn print_latency_summary(sent: u32, samples: &[Duration]) {
+    match latency_stats(sent, samples) {
+        Some(stats) => println!(
+            "latency summary: sent={} received={} loss={:.1}% min={} avg={} max={}",
+            stats.sent,
+            stats.received,
+            stats.loss_ratio * 100.0,
+            format_duration(stats.min),
+            format_duration(stats.avg),
+            format_duration(stats.max)
+        ),
+        None => println!("latency summary: no replies"),
+    }
+}
+
+fn print_capacity_round(round: u32, sample: CapacitySample, sender_span: Duration) {
+    let mbps = sample
+        .mbps()
+        .map_or_else(|| "n/a".to_owned(), |value| format!("{value:.1} Mbps"));
+    println!(
+        "capacity {:>3}: rx={}/{} loss={:.1}% span={} estimate={} sender_burst={}",
+        round,
+        sample.received_packets,
+        sample.sent_packets,
+        sample.loss_ratio() * 100.0,
+        format_duration(sample.span),
+        mbps,
+        format_duration(sender_span)
+    );
+}
+
+fn print_capacity_summary(samples: &[CapacitySample]) {
+    let mut estimates = samples
+        .iter()
+        .filter_map(|sample| sample.mbps())
+        .collect::<Vec<f64>>();
+    if estimates.is_empty() {
+        println!("capacity summary: no usable samples");
+        return;
+    }
+
+    estimates.sort_by(f64::total_cmp);
+    let median_index = estimates.len() / 2;
+    let median = estimates.get(median_index).copied().unwrap_or(0.0);
+    let best = estimates.last().copied().unwrap_or(median);
+    let received = samples
+        .iter()
+        .map(|sample| sample.received_packets)
+        .sum::<u32>();
+    let sent = samples
+        .iter()
+        .map(|sample| sample.sent_packets)
+        .sum::<u32>();
+    let loss = if sent == 0 {
+        0.0
+    } else {
+        f64::from(sent.saturating_sub(received)) / f64::from(sent)
+    };
+
+    println!(
+        "capacity summary: samples={} median={median:.1} Mbps best={best:.1} Mbps aggregate_loss={:.1}%",
+        estimates.len(),
+        loss * 100.0
+    );
+}
+
+fn parse_link_local_addr_arg(raw: &str) -> std::result::Result<Ipv6Addr, String> {
+    parse_link_local_addr(raw).map_err(|err| err.to_string())
+}
+
+fn is_timeout(err: &io::Error) -> bool {
+    matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut)
+}
+
+fn div_duration(duration: Duration, divisor: u32) -> Duration {
+    if divisor == 0 {
+        return duration;
+    }
+    Duration::from_nanos(duration_nanos_u64(duration) / u64::from(divisor))
+}
+
+fn format_duration(duration: Duration) -> String {
+    if duration < Duration::from_millis(1) {
+        return format!("{:.3} us", duration.as_secs_f64() * 1_000_000.0);
+    }
+    format!("{:.3} ms", duration.as_secs_f64() * 1_000.0)
+}
+
+fn make_base_run_id() -> u64 {
+    rand::random()
+}
+
+#[cfg(test)]
+mod tests {
+    use std::net::Ipv6Addr;
+
+    use clap::Parser;
+
+    use super::{Cli, Command, mark_seen};
+
+    #[test]
+    fn parses_probe_options() {
+        let cli = Cli::try_parse_from([
+            "link_profile",
+            "probe",
+            "--ifname",
+            "en3",
+            "--peer",
+            "fe80::1%en3",
+            "--train-packets",
+            "32",
+        ])
+        .expect("probe options should parse");
+
+        match cli.command {
+            Command::Probe(args) => {
+                let options = args.into_probe_options();
+                assert_eq!(options.ifname, "en3");
+                assert_eq!(
+                    options.peer,
+                    "fe80::1".parse::<Ipv6Addr>().expect("valid IPv6")
+                );
+                assert_eq!(options.config.train_packets, 32);
+            }
+            Command::Reflect(_) => panic!("expected probe command"),
+        }
+    }
+
+    #[test]
+    fn parses_reflect_options() {
+        let cli = Cli::try_parse_from([
+            "link_profile",
+            "reflect",
+            "--ifname",
+            "en2",
+            "--port",
+            "42000",
+        ])
+        .expect("reflect options should parse");
+
+        match cli.command {
+            Command::Reflect(options) => {
+                assert_eq!(options.ifname, "en2");
+                assert_eq!(options.port, 42_000);
+            }
+            Command::Probe(_) => panic!("expected reflect command"),
+        }
+    }
+
+    #[test]
+    fn mark_seen_accepts_each_sequence_once() {
+        let mut seen = vec![false; 2];
+
+        assert!(mark_seen(&mut seen, 0));
+        assert!(!mark_seen(&mut seen, 0));
+        assert!(mark_seen(&mut seen, 1));
+        assert!(!mark_seen(&mut seen, 2));
+    }
+}

rust/babblerd/src/profiling/types.rs

@@ -0,0 +1,49 @@
+use std::net::Ipv6Addr;
+use std::time::Duration;
+
+use crate::config::TUN_MTU;
+
+pub const DEFAULT_PROFILE_PORT: u16 = 41_901;
+pub const DEFAULT_ECHO_COUNT: u32 = 10;
+pub const DEFAULT_ECHO_INTERVAL_MS: u64 = 250;
+pub const DEFAULT_ECHO_TIMEOUT_MS: u64 = 500;
+pub const DEFAULT_CAPACITY_ROUNDS: u32 = 5;
+pub const DEFAULT_TRAIN_PACKETS: u32 = 64;
+pub const DEFAULT_TRAIN_INTERVAL_MS: u64 = 1_000;
+pub const DEFAULT_TRAIN_SETTLE_MS: u64 = 25;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LinkKey {
+    pub ifname: Box<str>,
+    pub ifindex: u32,
+    pub peer_link_local: Ipv6Addr,
+}
+
+#[derive(Debug, Clone)]
+pub struct ProbeConfig {
+    pub port: u16,
+    pub echo_count: u32,
+    pub echo_interval: Duration,
+    pub echo_timeout: Duration,
+    pub capacity_rounds: u32,
+    pub train_packets: u32,
+    pub train_payload_bytes: usize,
+    pub train_interval: Duration,
+    pub train_settle: Duration,
+}
+
+impl Default for ProbeConfig {
+    fn default() -> Self {
+        Self {
+            port: DEFAULT_PROFILE_PORT,
+            echo_count: DEFAULT_ECHO_COUNT,
+            echo_interval: Duration::from_millis(DEFAULT_ECHO_INTERVAL_MS),
+            echo_timeout: Duration::from_millis(DEFAULT_ECHO_TIMEOUT_MS),
+            capacity_rounds: DEFAULT_CAPACITY_ROUNDS,
+            train_packets: DEFAULT_TRAIN_PACKETS,
+            train_payload_bytes: usize::from(TUN_MTU),
+            train_interval: Duration::from_millis(DEFAULT_TRAIN_INTERVAL_MS),
+            train_settle: Duration::from_millis(DEFAULT_TRAIN_SETTLE_MS),
+        }
+    }
+}

rust/babblerd/src/route_ctl.rs

@@ -0,0 +1,59 @@
+use std::io;
+use std::net::IpAddr;
+
+use ipnet::Ipv6Net;
+use nix::net::if_::if_nametoindex;
+use route_manager::{Route, RouteManager};
+
+use crate::Result;
+
+fn route_destination(prefix: Ipv6Net) -> IpAddr {
+    IpAddr::V6(prefix.trunc().addr())
+}
+
+fn is_overlay_route(route: &Route, prefix: Ipv6Net) -> bool {
+    route.destination() == route_destination(prefix) && route.prefix() == prefix.prefix_len()
+}
+
+fn tun_if_index(tun_ifname: &str) -> io::Result<u32> {
+    if_nametoindex(tun_ifname).map_err(io::Error::from)
+}
+
+pub fn ensure_overlay_route(prefix: Ipv6Net, tun_ifname: &str) -> Result<()> {
+    let tun_ifindex = tun_if_index(tun_ifname)?;
+    let desired =
+        Route::new(route_destination(prefix), prefix.prefix_len()).with_if_index(tun_ifindex);
+    let mut manager = RouteManager::new()?;
+    let existing: Vec<Route> = manager
+        .list()?
+        .into_iter()
+        .filter(|route| is_overlay_route(route, prefix))
+        .collect();
+
+    let already_present = existing
+        .iter()
+        .any(|route| route.if_index() == Some(tun_ifindex) && route.gateway().is_none());
+    if already_present {
+        return Ok(());
+    }
+
+    for route in existing {
+        manager.delete(&route)?;
+    }
+    manager.add(&desired)?;
+    Ok(())
+}
+
+pub fn remove_overlay_route(prefix: Ipv6Net) -> Result<()> {
+    let mut manager = RouteManager::new()?;
+    let existing: Vec<Route> = manager
+        .list()?
+        .into_iter()
+        .filter(|route| is_overlay_route(route, prefix))
+        .collect();
+
+    for route in existing {
+        manager.delete(&route)?;
+    }
+    Ok(())
+}

rust/babblerd/src/routing_stack.rs

@@ -0,0 +1,205 @@
+use std::sync::Arc;
+
+use color_eyre::eyre::{Result, WrapErr};
+use ipnet::Ipv6Net;
+use tokio::{
+    sync::{mpsc, watch},
+    task::JoinHandle,
+    time::{Duration, MissedTickBehavior},
+};
+
+use crate::babel::BabelState;
+use crate::config::TUN_MTU;
+use crate::daemon::{RoutingStackEvent, StackTaskKind};
+use crate::dataplane::{Dataplane, DataplaneConfig, DataplanePublisher, PublishSnapshotError};
+use crate::fib::FibBuilder;
+use crate::tun::TunDevice;
+
+pub struct RoutingStack {
+    babel: JoinHandle<crate::Result<()>>,
+    watcher: JoinHandle<crate::Result<()>>,
+    state_logger: JoinHandle<()>,
+    fib_publisher: JoinHandle<crate::Result<()>>,
+    dataplane_monitor: JoinHandle<()>,
+    dataplane: Dataplane,
+}
+
+impl RoutingStack {
+    pub fn start(
+        node_addr: Ipv6Net,
+        tun: &TunDevice,
+        udp_port: u16,
+        state_send: watch::Sender<Arc<BabelState>>,
+        event_send: mpsc::Sender<RoutingStackEvent>,
+    ) -> Result<Self> {
+        let (iface_send, iface_recv) = mpsc::channel(32);
+        let mut state_recv = state_send.subscribe();
+        let fib_state_recv = state_send.subscribe();
+        let initial_state = state_send.borrow().clone();
+        let mut dataplane = Dataplane::spawn(DataplaneConfig {
+            tun_device: tun.shared_device(),
+            udp_port,
+            initial_fib: Arc::new(
+                FibBuilder::new([node_addr.addr()], TUN_MTU).derive(initial_state.as_ref()),
+            ),
+        })?;
+        let dataplane_exit = dataplane
+            .take_exit_receiver()
+            .ok_or_else(|| color_eyre::eyre::eyre!("dataplane exit receiver missing"))?;
+
+        let state_logger = tokio::spawn(async move {
+            while state_recv.changed().await.is_ok() {
+                let snapshot = state_recv.borrow_and_update();
+                tracing::info!(state = ?*snapshot, "babel state snapshot updated");
+            }
+            tracing::info!("babel state stream closed");
+        });
+
+        let babel_events = event_send.clone();
+        let babel = tokio::spawn(async move {
+            let res = crate::babel(node_addr, iface_recv, state_send).await;
+            let _ = babel_events
+                .send(RoutingStackEvent::Exited {
+                    kind: StackTaskKind::Babel,
+                    error: res.as_ref().err().map(ToString::to_string),
+                })
+                .await;
+            res
+        });
+
+        let watcher_events = event_send.clone();
+        let watcher = tokio::spawn(async move {
+            let res = crate::watch(iface_send).await;
+            let _ = watcher_events
+                .send(RoutingStackEvent::Exited {
+                    kind: StackTaskKind::Watcher,
+                    error: res.as_ref().err().map(ToString::to_string),
+                })
+                .await;
+            res
+        });
+
+        let fib_events = event_send.clone();
+        let dataplane_publisher = dataplane.publisher();
+        let fib_publisher = tokio::spawn(async move {
+            let res = publish_fib_updates(node_addr, fib_state_recv, dataplane_publisher).await;
+            let _ = fib_events
+                .send(RoutingStackEvent::Exited {
+                    kind: StackTaskKind::FibPublisher,
+                    error: res.as_ref().err().map(ToString::to_string),
+                })
+                .await;
+            res
+        });
+
+        let dataplane_events = event_send;
+        let dataplane_monitor = tokio::spawn(async move {
+            let exit = dataplane_exit.await;
+            let (kind, error) = match exit {
+                Ok(Ok(())) => (StackTaskKind::Dataplane, None),
+                Ok(Err(err)) => (StackTaskKind::Dataplane, Some(err)),
+                Err(err) => (
+                    StackTaskKind::Dataplane,
+                    Some(format!("dataplane exit receiver dropped: {err}")),
+                ),
+            };
+            let _ = dataplane_events
+                .send(RoutingStackEvent::Exited { kind, error })
+                .await;
+        });
+
+        Ok(Self {
+            babel,
+            watcher,
+            state_logger,
+            fib_publisher,
+            dataplane_monitor,
+            dataplane,
+        })
+    }
+
+    pub async fn stop(self) -> Result<()> {
+        let Self {
+            babel,
+            watcher,
+            state_logger,
+            fib_publisher,
+            dataplane_monitor,
+            dataplane,
+        } = self;
+
+        watcher.abort();
+        if let Ok(res) = watcher.await {
+            res.wrap_err("stopping interface watcher")?;
+        }
+
+        state_logger.abort();
+        let _ = state_logger.await;
+
+        fib_publisher.abort();
+        let _ = fib_publisher.await;
+
+        dataplane_monitor.abort();
+        let _ = dataplane_monitor.await;
+
+        dataplane.stop().wrap_err("stopping dataplane thread")?;
+
+        babel.await?.wrap_err("stopping babeld runtime")?;
+        Ok(())
+    }
+}
+
+async fn publish_fib_updates(
+    node_addr: Ipv6Net,
+    mut state_recv: watch::Receiver<Arc<BabelState>>,
+    publisher: DataplanePublisher,
+) -> crate::Result<()> {
+    let builder = FibBuilder::new([node_addr.addr()], TUN_MTU);
+    let mut pending = Some(Arc::new(builder.derive(state_recv.borrow().as_ref())));
+    let mut published: Option<Arc<crate::fib::FibSnapshot>> = None;
+    let mut retry_tick = tokio::time::interval(Duration::from_millis(10));
+    retry_tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
+
+    loop {
+        if let Some(snapshot) = pending.take() {
+            let published_snapshot = Arc::clone(&snapshot);
+            match publisher.try_publish(snapshot) {
+                Ok(()) => {
+                    published = Some(published_snapshot);
+                }
+                Err(PublishSnapshotError::Full(snapshot)) => {
+                    pending = Some(snapshot);
+                }
+                Err(PublishSnapshotError::Stopped) => {
+                    return Err(crate::BabbleError::Other(
+                        "dataplane thread stopped".to_owned(),
+                    ));
+                }
+            }
+        }
+
+        tokio::select! {
+            changed = state_recv.changed() => {
+                match changed {
+                    Ok(()) => {
+                        let snapshot = {
+                            let state = state_recv.borrow_and_update();
+                            Arc::new(builder.derive(state.as_ref()))
+                        };
+                        let matches_published = published
+                            .as_ref()
+                            .is_some_and(|current| current.as_ref() == snapshot.as_ref());
+                        let matches_pending = pending
+                            .as_ref()
+                            .is_some_and(|current| current.as_ref() == snapshot.as_ref());
+                        if !matches_published && !matches_pending {
+                            pending = Some(snapshot);
+                        }
+                    }
+                    Err(_) => return Ok(()),
+                }
+            }
+            _ = retry_tick.tick(), if pending.is_some() => {}
+        }
+    }
+}

rust/babblerd/src/tun.rs

@@ -0,0 +1,63 @@
+use ipnet::Ipv6Net;
+use std::net::Ipv6Addr;
+use std::sync::Arc;
+use tun_rs::{DeviceBuilder, SyncDevice};
+
+use crate::config::TUN_MTU;
+
+#[cfg(target_os = "linux")]
+const DESIRED_TUN_NAME: &str = "exonet";
+
+/// Holds the TUN device open for the lifetime of the daemon.
+/// The interface disappears when this is dropped.
+pub struct TunDevice {
+    dev: Arc<SyncDevice>,
+    ifname: String,
+    node_addr: Ipv6Net, // TODO: we are only ever gonna install /128 subnets, maybe change to Ipv6Addr in future??
+}
+
+impl TunDevice {
+    pub fn create(node_addr: Ipv6Addr) -> crate::Result<Self> {
+        let builder = DeviceBuilder::new().ipv6(node_addr, 128u8).mtu(TUN_MTU);
+        #[cfg(target_os = "linux")]
+        let builder = builder.name(DESIRED_TUN_NAME);
+
+        let dev = builder
+            .with(|builder| {
+                builder.packet_information(false);
+
+                #[cfg(target_os = "macos")]
+                {
+                    // Route ownership stays in userspace; do not let tun-rs auto-add routes.
+                    // The IPv6 /128 address itself is still applied by tun-rs.
+                    builder.associate_route(false);
+                }
+            })
+            .build_sync()?;
+
+        dev.set_nonblocking(true)?;
+        let ifname = dev.name()?;
+
+        Ok(Self {
+            dev: Arc::new(dev),
+            ifname,
+            node_addr: Ipv6Net::new_assert(node_addr, 128), // TODO: i dont't like the magic numbers, I also don't like wrapping and unwrapping
+        })
+    }
+
+    pub fn ifname(&self) -> &str {
+        &self.ifname
+    }
+
+    pub fn node_addr(&self) -> Ipv6Net {
+        self.node_addr
+    }
+
+    pub fn device(&self) -> &SyncDevice {
+        self.dev.as_ref()
+    }
+
+    pub fn shared_device(&self) -> Arc<SyncDevice> {
+        Arc::clone(&self.dev)
+    }
+}

rust/exo_pyo3_bindings/Cargo.toml

@@ -0,0 +1,69 @@
+[package]
+name = "exo_pyo3_bindings"
+version = { workspace = true }
+edition = { workspace = true }
+publish = false
+
+[lib]
+doctest = false
+path = "src/lib.rs"
+name = "exo_pyo3_bindings"
+
+# "cdylib" needed to produce shared library for Python to import
+# "rlib" needed for stub-gen to run
+crate-type = ["cdylib", "rlib"]
+
+[[bin]]
+path = "src/bin/stub_gen.rs"
+name = "stub_gen"
+doc = false
+
+[lints]
+workspace = true
+
+[dependencies]
+networking = { workspace = true }
+
+# interop
+pyo3 = { version = "0.27.2", features = [
+  # "abi3-py313", # tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.13
+  # "nightly", # enables better-supported GIL integration
+  "experimental-async", # async support in #[pyfunction] & #[pymethods]
+  #"experimental-inspect", # inspection of generated binary => easier to automate type-hint generation
+  #"py-clone", # adding Clone-ing of `Py<T>` without GIL (may cause panics - remove if panics happen)
+  # "multiple-pymethods", # allows multiple #[pymethods] sections per class
+
+  # integrations with other libraries
+  # "arc_lock", "bigdecimal", "either", "hashbrown", "indexmap", "num-bigint", "num-complex", "num-rational",
+  # "ordered-float", "rust_decimal", "smallvec",
+  # "anyhow", "chrono", "chrono-local", "chrono-tz", "eyre", "jiff-02", "lock_api", "parking-lot", "time",  "serde",
+] }
+pyo3-stub-gen = { version = "0.17.2" }
+pyo3-async-runtimes = { version = "0.27.0", features = [
+  "attributes",
+  "tokio-runtime",
+  "testing",
+] }
+pyo3-log = "0.13.2"
+
+pidfile-rs = "0.3"
+
+# macro dependencies
+extend = { workspace = true }
+delegate = { workspace = true }
+thiserror = "2.0"
+
+# async runtime
+tokio = { workspace = true, features = ["full", "tracing"] }
+futures-lite = { workspace = true }
+
+# utility dependencies
+util = { workspace = true }
+
+# Tracing
+log = { workspace = true }
+env_logger = "0.11"
+
+# Networking
+libp2p = { workspace = true, features = ["full"] }
+pin-project = "1.1.10"

rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi

@@ -1,41 +1,50 @@
 # This file is automatically generated by pyo3_stub_gen
-# ruff: noqa: E501, F401, F403, F405
+# ruff: noqa: E501, F401
 
 import builtins
 import os
 import pathlib
 import typing
-__all__ = [
-    "FromSwarm",
-    "NetworkingHandle",
-    "Pidfile",
-    "PidfileError",
-]
 
-class FromSwarm:
-    @typing.final
-    class Connection(FromSwarm):
-        __match_args__ = ("connected",)
-        @property
-        def connected(self) -> builtins.bool: ...
-        def __new__(cls, connected: builtins.bool) -> FromSwarm.Connection: ...
-    
-    @typing.final
-    class Message(FromSwarm):
-        __match_args__ = ("topic", "data",)
-        @property
-        def topic(self) -> builtins.str: ...
-        @property
-        def data(self) -> bytes: ...
-        def __new__(cls, topic: builtins.str, data: bytes) -> FromSwarm.Message: ...
-    
-    ...
+@typing.final
+class AllQueuesFullError(builtins.Exception):
+    def __new__(cls, *args: typing.Any) -> AllQueuesFullError: ...
+    def __repr__(self) -> builtins.str: ...
+    def __str__(self) -> builtins.str: ...
+
+@typing.final
+class Keypair:
+    r"""
+    Identity keypair of a node.
+    """
+    @staticmethod
+    def generate() -> Keypair:
+        r"""
+        Generate a new Ed25519 keypair.
+        """
+    @staticmethod
+    def from_bytes(bytes: bytes) -> Keypair:
+        r"""
+        Construct an Ed25519 keypair from secret key bytes
+        """
+    def to_bytes(self) -> bytes:
+        r"""
+        Get the secret key bytes underlying the keypair
+        """
+    def to_node_id(self) -> builtins.str:
+        r"""
+        Convert the `Keypair` into the corresponding `PeerId` string, which we use as our `NodeId`.
+        """
+
+@typing.final
+class MessageTooLargeError(builtins.Exception):
+    def __new__(cls, *args: typing.Any) -> MessageTooLargeError: ...
+    def __repr__(self) -> builtins.str: ...
+    def __str__(self) -> builtins.str: ...
 
 @typing.final
 class NetworkingHandle:
-    @staticmethod
-    def new(identity: builtins.str, namespace: builtins.str, listen_port: builtins.int, discovery_service_port: builtins.int) -> NetworkingHandle: ...
-    def recv(self) -> typing.Awaitable[FromSwarm]: ...
+    def __new__(cls, identity: Keypair, bootstrap_peers: typing.Sequence[builtins.str], listen_port: builtins.int) -> NetworkingHandle: ...
     async def gossipsub_subscribe(self, topic: builtins.str) -> builtins.bool:
         r"""
         Subscribe to a `GossipSub` topic.
@@ -54,6 +63,13 @@ class NetworkingHandle:
         
         If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception.
         """
+    async def recv(self) -> PyFromSwarm: ...
+
+@typing.final
+class NoPeersSubscribedToTopicError(builtins.Exception):
+    def __new__(cls, *args: typing.Any) -> NoPeersSubscribedToTopicError: ...
+    def __repr__(self) -> builtins.str: ...
+    def __str__(self) -> builtins.str: ...
 
 @typing.final
 class Pidfile:
@@ -61,7 +77,7 @@ class Pidfile:
     A PID file protected with a lock.
     
     An instance of `Pidfile` can be used to manage a PID file: create it,
-    lock it, detect already running daemons. It is backed by [`pidfile`]
+    lock it, detect already running daemons. It is backed by [`pidfile`][]
     functions of `libbsd`/`libutil` which use `flopen` to lock the PID
     file.
     
@@ -91,23 +107,32 @@ class Pidfile:
         
         The file is truncated before writing.
         """
-    def as_raw_fd(self) -> builtins.int:
-        r"""
-        Extracts the raw file descriptor.
-        
-        This function is typically used to **borrow** an owned file descriptor.
-        When used in this way, this method does **not** pass ownership of the
-        raw file descriptor to the caller, and the file descriptor is only
-        guaranteed to be valid while the original object has not yet been
-        destroyed.
-        """
-    def close(self) -> None:
-        r"""
-        Closes the PID file and releases associated resources.
-        """
 
 @typing.final
 class PidfileError(builtins.Exception):
     def __repr__(self) -> builtins.str: ...
     def __str__(self) -> builtins.str: ...
 
+class PyFromSwarm:
+    @typing.final
+    class Connection(PyFromSwarm):
+        __match_args__ = ("peer_id", "connected",)
+        @property
+        def peer_id(self) -> builtins.str: ...
+        @property
+        def connected(self) -> builtins.bool: ...
+        def __new__(cls, peer_id: builtins.str, connected: builtins.bool) -> PyFromSwarm.Connection: ...
+    
+    @typing.final
+    class Message(PyFromSwarm):
+        __match_args__ = ("origin", "topic", "data",)
+        @property
+        def origin(self) -> builtins.str: ...
+        @property
+        def topic(self) -> builtins.str: ...
+        @property
+        def data(self) -> bytes: ...
+        def __new__(cls, origin: builtins.str, topic: builtins.str, data: bytes) -> PyFromSwarm.Message: ...
+    
+    ...
+

rust/exo_pyo3_bindings/pyproject.toml

@@ -3,31 +3,27 @@ requires = ["maturin>=1.0,<2.0"]
 build-backend = "maturin"
 
 [project]
-name = "exo_rs"
-version = "0.3.0"
+name = "exo_pyo3_bindings"
+version = "0.2.2"
 description = "Add your description here"
 readme = "README.md"
 authors = [
-  { name = "Evan Quiney", email = "evanev7@gmail.com" },
   { name = "Andrei Cravtov", email = "the.andrei.cravtov@gmail.com" },
+  { name = "Evan Quiney", email = "evanev7@gmail.com" },
 ]
 requires-python = ">=3.13"
 dependencies = []
 
 [dependency-groups]
-dev = ["exo_rs", "pytest>=8.4.0", "pytest-asyncio>=1.0.0"]
+dev = ["exo_pyo3_bindings", "pytest>=8.4.0", "pytest-asyncio>=1.0.0"]
 
 [tool.maturin]
-module-name = "exo_rs"
+#purelib = true
+#python-source = "python"
+module-name = "exo_pyo3_bindings"
 features = ["pyo3/extension-module", "pyo3/experimental-async"]
 
-[tool.pyo3-stub-gen]
-generate-init-py = true
-
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-
-[tool.uv]
-cache-keys = [{ file = "src/**/*.rs" }]

rust/exo_pyo3_bindings/src/bin/stub_gen.rs

@@ -2,7 +2,7 @@ use pyo3_stub_gen::Result;
 
 fn main() -> Result<()> {
     env_logger::Builder::from_env(env_logger::Env::default().filter_or("RUST_LOG", "info")).init();
-    let stub = exo_rs::stub_info()?;
+    let stub = exo_pyo3_bindings::stub_info()?;
     stub.generate()?;
     Ok(())
 }

rust/exo_pyo3_bindings/src/ident.rs

@@ -0,0 +1,47 @@
+use crate::ext::ResultExt as _;
+use libp2p::identity::Keypair;
+use pyo3::types::{PyBytes, PyBytesMethods as _};
+use pyo3::{Bound, PyResult, Python, pyclass, pymethods};
+use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods};
+
+/// Identity keypair of a node.
+#[gen_stub_pyclass]
+#[pyclass(name = "Keypair", frozen)]
+#[repr(transparent)]
+pub struct PyKeypair(pub Keypair);
+
+#[gen_stub_pymethods]
+#[pymethods]
+#[allow(clippy::needless_pass_by_value)]
+impl PyKeypair {
+    /// Generate a new Ed25519 keypair.
+    #[staticmethod]
+    fn generate() -> Self {
+        Self(Keypair::generate_ed25519())
+    }
+
+    /// Construct an Ed25519 keypair from secret key bytes
+    #[staticmethod]
+    fn from_bytes(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
+        let mut bytes = Vec::from(bytes.as_bytes());
+        Ok(Self(Keypair::ed25519_from_bytes(&mut bytes).pyerr()?))
+    }
+
+    /// Get the secret key bytes underlying the keypair
+    fn to_bytes<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
+        let bytes = self
+            .0
+            .clone()
+            .try_into_ed25519()
+            .pyerr()?
+            .secret()
+            .as_ref()
+            .to_vec();
+        Ok(PyBytes::new(py, &bytes))
+    }
+
+    /// Convert the `Keypair` into the corresponding `PeerId` string, which we use as our `NodeId`.
+    fn to_node_id(&self) -> String {
+        self.0.public().to_peer_id().to_base58()
+    }
+}

rust/exo_pyo3_bindings/src/lib.rs

@@ -5,16 +5,23 @@
 //!
 
 mod allow_threading;
-// mod ident;
+mod ident;
 mod networking;
 mod pidfile;
 
+use crate::ident::PyKeypair;
 use crate::networking::networking_submodule;
 use crate::pidfile::pidfile_submodule;
 use pyo3::prelude::PyModule;
-use pyo3::{Bound, PyResult, pymodule};
+use pyo3::types::PyModuleMethods;
+use pyo3::{Bound, PyResult, pyclass, pymodule};
 use pyo3_stub_gen::define_stub_info_gatherer;
 
+/// Namespace for all the constants used by this crate.
+pub(crate) mod r#const {
+    pub const MPSC_CHANNEL_SIZE: usize = 1024;
+}
+
 /// Namespace for crate-wide extension traits/methods
 pub(crate) mod ext {
     use crate::allow_threading::AllowThreads;
@@ -146,7 +153,7 @@ pub(crate) mod ext {
 /// A Python module implemented in Rust. The name of this function must match
 /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
 /// import the module.
-#[pymodule(name = "exo_rs", gil_used = true)]
+#[pymodule(name = "exo_pyo3_bindings")]
 fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     // install logger
     pyo3_log::init();
@@ -157,9 +164,9 @@ fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     // TODO: for now this is all NOT a submodule, but figure out how to make the submodule system
     //       work with maturin, where the types generate correctly, in the right folder, without
     //       too many importing issues...
-    pidfile_submodule(m)?;
-    // m.add_class::<PyKeypair>()?;
+    m.add_class::<PyKeypair>()?;
     networking_submodule(m)?;
+    pidfile_submodule(m)?;
 
     // top-level constructs
     // TODO: ...

rust/exo_pyo3_bindings/src/networking.rs

@@ -0,0 +1,318 @@
+use std::pin::Pin;
+use std::sync::Arc;
+
+use crate::r#const::MPSC_CHANNEL_SIZE;
+use crate::ext::{ByteArrayExt as _, FutureExt, PyErrExt as _};
+use crate::ext::{ResultExt as _, TokioMpscSenderExt as _};
+use crate::ident::PyKeypair;
+use crate::networking::exception::{
+    PyAllQueuesFullError, PyMessageTooLargeError, PyNoPeersSubscribedToTopicError,
+};
+use crate::pyclass;
+use futures_lite::{Stream, StreamExt as _};
+use libp2p::gossipsub::PublishError;
+use networking::swarm::{FromSwarm, ToSwarm, create_swarm};
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::{PyModule, PyModuleMethods as _};
+use pyo3::types::PyBytes;
+use pyo3::{Bound, Py, PyAny, PyErr, PyResult, Python, pymethods};
+use pyo3_stub_gen::derive::{
+    gen_methods_from_python, gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods,
+};
+use tokio::sync::{Mutex, mpsc, oneshot};
+
+mod exception {
+    use pyo3::types::PyTuple;
+    use pyo3::{exceptions::PyException, prelude::*};
+    use pyo3_stub_gen::derive::*;
+
+    #[gen_stub_pyclass]
+    #[pyclass(frozen, extends=PyException, name="NoPeersSubscribedToTopicError")]
+    pub struct PyNoPeersSubscribedToTopicError {}
+
+    impl PyNoPeersSubscribedToTopicError {
+        const MSG: &'static str = "\
+        No peers are currently subscribed to receive messages on this topic. \
+        Wait for peers to subscribe or check your network connectivity.";
+
+        ///   Creates a new  [ `PyErr` ]  of this type.
+        ///
+        ///   [`PyErr`] :  https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html   "PyErr in pyo3"
+        pub(crate) fn new_err() -> PyErr {
+            PyErr::new::<Self, _>(()) // TODO: check if this needs to be replaced???
+        }
+    }
+
+    #[gen_stub_pymethods]
+    #[pymethods]
+    impl PyNoPeersSubscribedToTopicError {
+        #[new]
+        #[pyo3(signature = (*args))]
+        #[allow(unused_variables)]
+        pub(crate) fn new(args: &Bound<'_, PyTuple>) -> Self {
+            Self {}
+        }
+
+        fn __repr__(&self) -> String {
+            format!("PeerId(\"{}\")", Self::MSG)
+        }
+
+        fn __str__(&self) -> String {
+            Self::MSG.to_string()
+        }
+    }
+
+    #[gen_stub_pyclass]
+    #[pyclass(frozen, extends=PyException, name="AllQueuesFullError")]
+    pub struct PyAllQueuesFullError {}
+
+    impl PyAllQueuesFullError {
+        const MSG: &'static str =
+            "All libp2p peers are unresponsive, resend the message or reconnect.";
+
+        ///   Creates a new  [ `PyErr` ]  of this type.
+        ///
+        ///   [`PyErr`] :  https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html   "PyErr in pyo3"
+        pub(crate) fn new_err() -> PyErr {
+            PyErr::new::<Self, _>(()) // TODO: check if this needs to be replaced???
+        }
+    }
+
+    #[gen_stub_pymethods]
+    #[pymethods]
+    impl PyAllQueuesFullError {
+        #[new]
+        #[pyo3(signature = (*args))]
+        #[allow(unused_variables)]
+        pub(crate) fn new(args: &Bound<'_, PyTuple>) -> Self {
+            Self {}
+        }
+
+        fn __repr__(&self) -> String {
+            format!("PeerId(\"{}\")", Self::MSG)
+        }
+
+        fn __str__(&self) -> String {
+            Self::MSG.to_string()
+        }
+    }
+
+    #[gen_stub_pyclass]
+    #[pyclass(frozen, extends=PyException, name="MessageTooLargeError")]
+    pub struct PyMessageTooLargeError {}
+
+    impl PyMessageTooLargeError {
+        const MSG: &'static str = "Gossipsub message exceeds max_transmit_size. Reduce prompt length or increase the limit.";
+
+        pub(crate) fn new_err() -> PyErr {
+            PyErr::new::<Self, _>(())
+        }
+    }
+
+    #[gen_stub_pymethods]
+    #[pymethods]
+    impl PyMessageTooLargeError {
+        #[new]
+        #[pyo3(signature = (*args))]
+        #[allow(unused_variables)]
+        pub(crate) fn new(args: &Bound<'_, PyTuple>) -> Self {
+            Self {}
+        }
+
+        fn __repr__(&self) -> String {
+            format!("MessageTooLargeError(\"{}\")", Self::MSG)
+        }
+
+        fn __str__(&self) -> String {
+            Self::MSG.to_string()
+        }
+    }
+}
+
+#[gen_stub_pyclass]
+#[pyclass(name = "NetworkingHandle")]
+struct PyNetworkingHandle {
+    // channels
+    pub to_swarm: mpsc::Sender<ToSwarm>,
+    pub swarm: Arc<Mutex<Pin<Box<dyn Stream<Item = FromSwarm> + Send>>>>,
+}
+
+#[gen_stub_pyclass_complex_enum]
+#[pyclass]
+enum PyFromSwarm {
+    Connection {
+        peer_id: String,
+        connected: bool,
+    },
+    Message {
+        origin: String,
+        topic: String,
+        data: Py<PyBytes>,
+    },
+}
+impl From<FromSwarm> for PyFromSwarm {
+    fn from(value: FromSwarm) -> Self {
+        match value {
+            FromSwarm::Discovered { peer_id } => Self::Connection {
+                peer_id: peer_id.to_base58(),
+                connected: true,
+            },
+            FromSwarm::Expired { peer_id } => Self::Connection {
+                peer_id: peer_id.to_base58(),
+                connected: false,
+            },
+            FromSwarm::Message { from, topic, data } => Self::Message {
+                origin: from.to_base58(),
+                topic: topic,
+                data: data.pybytes(),
+            },
+        }
+    }
+}
+
+#[gen_stub_pymethods]
+#[pymethods]
+impl PyNetworkingHandle {
+    // NOTE: `async fn`s here that use `.await` will wrap the future in `.allow_threads_py()`
+    //       immediately beforehand to release the interpreter.
+    //       SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
+
+    // ---- Lifecycle management methods ----
+
+    #[new]
+    #[pyo3(signature = (identity, bootstrap_peers, listen_port))]
+    fn py_new(
+        identity: Bound<'_, PyKeypair>,
+        bootstrap_peers: Vec<String>,
+        listen_port: u16,
+    ) -> PyResult<Self> {
+        // create communication channels
+        let (to_swarm, from_client) = mpsc::channel(MPSC_CHANNEL_SIZE);
+
+        // get identity
+        let identity = identity.borrow().0.clone();
+
+        // create networking swarm (within tokio context!! or it crashes)
+        let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
+        let swarm = create_swarm(identity, from_client, bootstrap_peers, listen_port)
+            .pyerr()?
+            .into_stream();
+
+        Ok(Self {
+            swarm: Arc::new(Mutex::new(swarm)),
+            to_swarm,
+        })
+    }
+
+    #[gen_stub(skip)]
+    fn recv<'py>(&'py self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let swarm = Arc::clone(&self.swarm);
+        pyo3_async_runtimes::tokio::future_into_py(py, async move {
+            swarm
+                .try_lock()
+                .map_err(|_| PyRuntimeError::new_err("called recv twice concurrently"))?
+                .next()
+                .await
+                .ok_or(PyErr::receiver_channel_closed())
+                .map(PyFromSwarm::from)
+        })
+    }
+
+    // ---- Gossipsub management methods ----
+
+    /// Subscribe to a `GossipSub` topic.
+    ///
+    /// Returns `True` if the subscription worked. Returns `False` if we were already subscribed.
+    async fn gossipsub_subscribe(&self, topic: String) -> PyResult<bool> {
+        let (tx, rx) = oneshot::channel();
+
+        // send off request to subscribe
+        self.to_swarm
+            .send_py(ToSwarm::Subscribe {
+                topic,
+                result_sender: tx,
+            })
+            .allow_threads_py() // allow-threads-aware async call
+            .await?;
+
+        // wait for response & return any errors
+        rx.allow_threads_py() // allow-threads-aware async call
+            .await
+            .map_err(|_| PyErr::receiver_channel_closed())?
+            .pyerr()
+    }
+
+    /// Unsubscribes from a `GossipSub` topic.
+    ///
+    /// Returns `True` if we were subscribed to this topic. Returns `False` if we were not subscribed.
+    async fn gossipsub_unsubscribe(&self, topic: String) -> PyResult<bool> {
+        let (tx, rx) = oneshot::channel();
+
+        // send off request to unsubscribe
+        self.to_swarm
+            .send_py(ToSwarm::Unsubscribe {
+                topic,
+                result_sender: tx,
+            })
+            .allow_threads_py() // allow-threads-aware async call
+            .await?;
+
+        // wait for response & convert any errors
+        rx.allow_threads_py() // allow-threads-aware async call
+            .await
+            .map_err(|_| PyErr::receiver_channel_closed())
+    }
+
+    /// Publishes a message with multiple topics to the `GossipSub` network.
+    ///
+    /// If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception.
+    async fn gossipsub_publish(&self, topic: String, data: Py<PyBytes>) -> PyResult<()> {
+        let (tx, rx) = oneshot::channel();
+
+        // send off request to subscribe
+        let data = Python::attach(|py| Vec::from(data.as_bytes(py)));
+        self.to_swarm
+            .send_py(ToSwarm::Publish {
+                topic,
+                data,
+                result_sender: tx,
+            })
+            .allow_threads_py() // allow-threads-aware async call
+            .await?;
+
+        // wait for response & return any errors => ignore messageID for now!!!
+        let _ = rx
+            .allow_threads_py() // allow-threads-aware async call
+            .await
+            .map_err(|_| PyErr::receiver_channel_closed())?
+            .map_err(|e| match e {
+                PublishError::AllQueuesFull(_) => PyAllQueuesFullError::new_err(),
+                PublishError::MessageTooLarge => PyMessageTooLargeError::new_err(),
+                PublishError::NoPeersSubscribedToTopic => {
+                    PyNoPeersSubscribedToTopicError::new_err()
+                }
+                e => PyRuntimeError::new_err(e.to_string()),
+            })?;
+        Ok(())
+    }
+}
+
+pyo3_stub_gen::inventory::submit! {
+    gen_methods_from_python! {
+        r#"
+            class PyNetworkingHandle:
+                async def recv() -> PyFromSwarm: ...
+        "#
+    }
+}
+
+pub fn networking_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<exception::PyNoPeersSubscribedToTopicError>()?;
+    m.add_class::<exception::PyAllQueuesFullError>()?;
+    m.add_class::<exception::PyMessageTooLargeError>()?;
+
+    m.add_class::<PyNetworkingHandle>()?;
+    m.add_class::<PyFromSwarm>()?;
+
+    Ok(())
+}

rust/exo_pyo3_bindings/src/pidfile.rs

@@ -3,9 +3,7 @@ use pyo3::exceptions::PyException;
 use pyo3::prelude::{PyModule, PyModuleMethods};
 use pyo3::{Bound, PyErr, PyResult, Python, pyclass, pymethods};
 use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods};
-use std::fs;
 use std::fs::Permissions;
-use std::os::fd::{AsRawFd, RawFd};
 use std::os::unix::prelude::PermissionsExt;
 use std::path::PathBuf;
 
@@ -38,7 +36,7 @@ impl PyPidfileError {
 /// A PID file protected with a lock.
 ///
 /// An instance of `Pidfile` can be used to manage a PID file: create it,
-/// lock it, detect already running daemons. It is backed by [`pidfile`]
+/// lock it, detect already running daemons. It is backed by [`pidfile`][]
 /// functions of `libbsd`/`libutil` which use `flopen` to lock the PID
 /// file.
 ///
@@ -55,23 +53,7 @@ impl PyPidfileError {
 /// [`daemon`(3)]: https://linux.die.net/man/3/daemon
 #[gen_stub_pyclass]
 #[pyclass(name = "Pidfile")]
-pub struct PyPidfile(Option<Pidfile>);
-
-impl PyPidfile {
-    #[inline(always)]
-    fn get(&self) -> &Pidfile {
-        self.0
-            .as_ref()
-            .expect("cannot use resource after exiting context")
-    }
-
-    #[inline(always)]
-    fn get_mut(&mut self) -> &mut Pidfile {
-        self.0
-            .as_mut()
-            .expect("cannot use resource after exiting context")
-    }
-}
+pub struct PyPidfile(Pidfile);
 
 #[gen_stub_pymethods]
 #[pymethods]
@@ -83,40 +65,17 @@ impl PyPidfile {
     /// the PID file yet.
     #[new]
     fn py_new(py: Python, path: PathBuf, mode: u32) -> PyResult<Self> {
-        // create all parent directories if don't exist
-        if let Some(parent) = path.parent() {
-            fs::create_dir_all(parent)
-                .map_err(|e| PyPidfileError(PidfileError::Io(e)).into_pyerr(py))?;
-        }
-
-        let pidfile = Pidfile::new(&path, Permissions::from_mode(mode))
-            .map_err(|e| PyPidfileError(e).into_pyerr(py))?;
-        Ok(Self(Some(pidfile)))
+        Ok(Self(
+            Pidfile::new(&path, Permissions::from_mode(mode))
+                .map_err(|e| PyPidfileError(e).into_pyerr(py))?,
+        ))
     }
 
     /// Writes the current process ID to the PID file.
     ///
     /// The file is truncated before writing.
     fn write<'py>(&mut self, py: Python<'py>) -> PyResult<()> {
-        self.get_mut()
-            .write()
-            .map_err(|e| PyPidfileError(e).into_pyerr(py))
-    }
-
-    /// Extracts the raw file descriptor.
-    ///
-    /// This function is typically used to **borrow** an owned file descriptor.
-    /// When used in this way, this method does **not** pass ownership of the
-    /// raw file descriptor to the caller, and the file descriptor is only
-    /// guaranteed to be valid while the original object has not yet been
-    /// destroyed.
-    fn as_raw_fd(&self) -> RawFd {
-        self.get().as_raw_fd()
-    }
-
-    /// Closes the PID file and releases associated resources.
-    fn close(&mut self) {
-        self.0 = None;
+        self.0.write().map_err(|e| PyPidfileError(e).into_pyerr(py))
     }
 }
 

rust/exo_pyo3_bindings/tests/test_python.py

@@ -1,28 +1,31 @@
 import asyncio
-import os
 
 import pytest
 from _pytest.capture import CaptureFixture
-from exo_rs import (
+from exo_pyo3_bindings import (
+    Keypair,
     NetworkingHandle,
+    NoPeersSubscribedToTopicError,
     Pidfile,
-    FromSwarm,
+    PyFromSwarm,
 )
 
 
 @pytest.mark.asyncio
 async def test_sleep_on_multiple_items() -> None:
     print("PYTHON: starting handle")
-    h = NetworkingHandle.new(os.urandom(16).hex().lstrip("0"), 52414, 52413)
-    print("PYTHON: handle started")
+    h = NetworkingHandle(Keypair.generate(), [], 0)
 
     rt = asyncio.create_task(_await_recv(h))
 
     # sleep for 4 ticks
-    for i in range(10):
+    for i in range(4):
         await asyncio.sleep(1)
 
-        await h.gossipsub_publish("topic", b"somehting or other")
+        try:
+            await h.gossipsub_publish("topic", b"somehting or other")
+        except NoPeersSubscribedToTopicError as e:
+            print("caught it", e)
 
 
 def test_pidfile(capsys: CaptureFixture[str]):
@@ -36,15 +39,11 @@ async def _await_recv(h: NetworkingHandle):
     while True:
         event = await h.recv()
         match event:
-            case FromSwarm.Connection() as c:
+            case PyFromSwarm.Connection() as c:
                 print(f"PYTHON: connection update: {c}")
-            case FromSwarm.Message() as m:
+            case PyFromSwarm.Message() as m:
                 print(f"PYTHON: message: {m}")
 
 
 def scoped_lock_file():
     a = Pidfile("/tmp/lock.pid", 0o0600)
-
-
-if __name__ == "__main__":
-    asyncio.run(test_sleep_on_multiple_items())

rust/exo_rs/Cargo.toml

@@ -1,53 +0,0 @@
-[package]
-name = "exo_rs"
-version = { workspace = true }
-edition = { workspace = true }
-publish = false
-
-[lib]
-doctest = false
-path = "src/lib.rs"
-name = "exo_rs"
-
-# "cdylib" needed to produce shared library for Python to import
-# "rlib" needed for stub-gen to run
-crate-type = ["cdylib", "rlib"]
-
-[[bin]]
-path = "src/bin/stub_gen.rs"
-name = "stub_gen"
-doc = false
-
-[lints]
-workspace = true
-
-[dependencies]
-networking.workspace = true
-extend.workspace = true
-
-# interop
-pyo3 = { workspace = true, features = ["experimental-async"] }
-pyo3-stub-gen.workspace = true
-pyo3-async-runtimes = { workspace = true, features = [
-  "attributes",
-  "tokio-runtime",
-  "testing",
-] }
-pyo3-log.workspace = true
-
-pidfile-rs = { workspace = true }
-
-# async runtime
-tokio = { workspace = true, features = ["full"] }
-futures-lite.workspace = true
-pin-project.workspace = true
-
-# Tracing
-log.workspace = true
-env_logger.workspace = true
-
-# Networking
-zenoh.workspace = true
-rand.workspace = true
-serde_json.workspace = true
-parking_lot.workspace = true

rust/exo_rs/src/networking.rs

@@ -1,197 +0,0 @@
-use std::pin::Pin;
-use std::sync::Arc;
-
-use crate::ext::{ByteArrayExt as _, FutureExt, PyErrExt as _};
-use crate::ext::{ResultExt as _, TokioMpscSenderExt as _};
-use futures_lite::{Stream, StreamExt as _};
-use networking::swarm::{FromSwarm, Swarm, ToSwarm, create_swarm};
-use networking::{Session, is_valid_zid};
-use pyo3::exceptions::{PyRuntimeError, PyValueError};
-use pyo3::prelude::*;
-use pyo3::types::PyBytes;
-use pyo3::{Bound, Py, PyAny, PyErr, PyResult, Python, pymethods};
-use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods};
-use tokio::sync::{Mutex, mpsc, oneshot};
-
-#[gen_stub_pyclass]
-#[pyclass(name = "NetworkingHandle")]
-pub struct PyNetworkingHandle {
-    // channels
-    pub to_swarm: mpsc::Sender<ToSwarm>,
-    pub swarm: Arc<Mutex<Pin<Box<dyn Stream<Item = FromSwarm> + Send>>>>,
-}
-
-#[gen_stub_pyclass_complex_enum]
-#[pyclass(name = "FromSwarm")]
-pub enum PyFromSwarm {
-    Connection { connected: bool },
-    Message { topic: String, data: Py<PyBytes> },
-}
-impl From<FromSwarm> for PyFromSwarm {
-    fn from(value: FromSwarm) -> Self {
-        match value {
-            FromSwarm::Discovered {} => Self::Connection { connected: true },
-            FromSwarm::Expired {} => Self::Connection { connected: false },
-            FromSwarm::Message { topic, data } => Self::Message {
-                topic: topic,
-                data: data.pybytes(),
-            },
-        }
-    }
-}
-
-impl PyNetworkingHandle {
-    pub fn from_session(session: Session) -> Self {
-        let (to_swarm, from_client) = mpsc::channel(1024);
-        let swarm = Swarm {
-            from_client,
-            session,
-        };
-        PyNetworkingHandle {
-            swarm: Arc::new(Mutex::new(swarm.into_stream())),
-            to_swarm,
-        }
-    }
-}
-
-#[gen_stub_pymethods]
-#[pymethods]
-impl PyNetworkingHandle {
-    // NOTE: `async fn`s here that use `.await` will wrap the future in `.allow_threads_py()`
-    //       immediately beforehand to release the interpreter.
-    //       SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
-
-    // ---- Lifecycle management methods ----
-
-    #[staticmethod]
-    pub fn new(
-        identity: &str,
-        namespace: &str,
-        listen_port: u16,
-        discovery_service_port: u16,
-    ) -> PyResult<PyNetworkingHandle> {
-        // todo: zenoh self assigned peers
-        if listen_port == 0 {
-            todo!("cannot listen on port 0 yet");
-        }
-        // create communication channels
-        let (to_swarm, from_client) = mpsc::channel(1024);
-
-        // get identity
-        if !is_valid_zid(identity) {
-            return Err(PyValueError::new_err(format!(
-                "{identity} is not a valid zenoh identity"
-            )));
-        }
-
-        // create networking swarm (within tokio context!! or it crashes)
-        let swarm = pyo3_async_runtimes::tokio::get_runtime()
-            .block_on(create_swarm(
-                identity,
-                namespace,
-                from_client,
-                listen_port,
-                discovery_service_port,
-            ))
-            .pyerr()?;
-
-        Ok(PyNetworkingHandle {
-            swarm: Arc::new(Mutex::new(swarm.into_stream())),
-            to_swarm,
-        })
-    }
-
-    #[gen_stub(override_return_type(
-        type_repr="typing.Awaitable[FromSwarm]", imports=("typing")
-    ))]
-    pub fn recv<'py>(&'py self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
-        let swarm = Arc::clone(&self.swarm);
-        pyo3_async_runtimes::tokio::future_into_py(py, async move {
-            swarm
-                .try_lock()
-                .map_err(|_| PyRuntimeError::new_err("called recv twice concurrently"))?
-                .next()
-                .await
-                .ok_or(PyErr::receiver_channel_closed())
-                .map(PyFromSwarm::from)
-        })
-    }
-
-    // ---- Gossipsub management methods ----
-
-    /// Subscribe to a `GossipSub` topic.
-    ///
-    /// Returns `True` if the subscription worked. Returns `False` if we were already subscribed.
-    pub async fn gossipsub_subscribe(&self, topic: String) -> PyResult<bool> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to subscribe
-        self.to_swarm
-            .send_py(ToSwarm::Subscribe {
-                topic,
-                result_sender: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & return any errors
-        rx.allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())?
-            .pyerr()
-    }
-
-    /// Unsubscribes from a `GossipSub` topic.
-    ///
-    /// Returns `True` if we were subscribed to this topic. Returns `False` if we were not subscribed.
-    pub async fn gossipsub_unsubscribe(&self, topic: String) -> PyResult<bool> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to unsubscribe
-        self.to_swarm
-            .send_py(ToSwarm::Unsubscribe {
-                topic,
-                result_sender: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & convert any errors
-        rx.allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())
-    }
-
-    /// Publishes a message with multiple topics to the `GossipSub` network.
-    ///
-    /// If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception.
-    pub async fn gossipsub_publish(&self, topic: String, data: Py<PyBytes>) -> PyResult<()> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to subscribe
-        let data = Python::attach(|py| Vec::from(data.as_bytes(py)));
-        self.to_swarm
-            .send_py(ToSwarm::Publish {
-                topic,
-                data,
-                result_sender: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & return any errors => ignore messageID for now!!!
-        let _ = rx
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())?
-            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
-        Ok(())
-    }
-}
-
-pub fn networking_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<PyNetworkingHandle>()?;
-    m.add_class::<PyFromSwarm>()?;
-
-    Ok(())
-}

rust/networking/Cargo.toml

@@ -1,27 +1,42 @@
 [package]
 name = "networking"
-version.workspace = true
-edition.workspace = true
+version = { workspace = true }
+edition = { workspace = true }
+publish = false
 
-[dependencies]
-async-stream.workspace = true
-futures-lite.workspace = true
-netwatcher = { workspace = true, features = ["tokio"] }
-parking_lot.workspace = true
-tokio = { workspace = true, features = ["full"] }
-zenoh = { workspace = true, features = ["internal", "plugins", "unstable"] }
-zenoh-plugin-storage-manager.workspace = true
-zenoh-plugin-trait.workspace = true
-rand.workspace = true
-log.workspace = true
-bytemuck = { workspace = true, features = ["derive"] }
-socket2.workspace = true
-blake3.workspace = true
+[lib]
+doctest = false
+name = "networking"
+path = "src/lib.rs"
 
 [lints]
 workspace = true
 
-[dev-dependencies]
-env_logger.workspace = true
-smol.workspace = true
-tracing.workspace = true
+[dependencies]
+# datastructures
+either = { workspace = true }
+
+# macro dependencies
+extend = { workspace = true }
+delegate = { workspace = true }
+
+# async
+async-stream = { workspace = true }
+futures-lite = { workspace = true }
+futures-timer = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+
+# utility dependencies
+util = { workspace = true }
+tracing-subscriber = { version = "0.3.19", features = [
+  "default",
+  "env-filter",
+] }
+keccak-const = { workspace = true }
+
+# tracing/logging
+log = { workspace = true }
+
+# networking
+libp2p = { workspace = true, features = ["full"] }
+pin-project = "1.1.10"

rust/networking/examples/chatroom.rs

@@ -0,0 +1,86 @@
+use futures_lite::StreamExt;
+use libp2p::identity;
+use networking::swarm;
+use networking::swarm::{FromSwarm, ToSwarm};
+use tokio::sync::{mpsc, oneshot};
+use tokio::{io, io::AsyncBufReadExt as _};
+use tracing_subscriber::EnvFilter;
+use tracing_subscriber::filter::LevelFilter;
+
+#[tokio::main]
+async fn main() {
+    let _ = tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::from_default_env().add_directive(LevelFilter::INFO.into()))
+        .try_init();
+
+    let (to_swarm, from_client) = mpsc::channel(20);
+
+    // Configure swarm
+    let mut swarm = swarm::create_swarm(
+        identity::Keypair::generate_ed25519(),
+        from_client,
+        vec![],
+        0,
+    )
+    .expect("Swarm creation failed")
+    .into_stream();
+
+    // Create a Gossipsub topic & subscribe
+    let (tx, rx) = oneshot::channel();
+    _ = to_swarm
+        .send(ToSwarm::Subscribe {
+            topic: "test-net".to_string(),
+            result_sender: tx,
+        })
+        .await
+        .expect("should send");
+
+    // Read full lines from stdin
+    let mut stdin = io::BufReader::new(io::stdin()).lines();
+    println!("Enter messages via STDIN and they will be sent to connected peers using Gossipsub");
+
+    tokio::task::spawn(async move {
+        rx.await
+            .expect("tx not dropped")
+            .expect("subscribe shouldn't fail");
+        loop {
+            if let Ok(Some(line)) = stdin.next_line().await {
+                let (tx, rx) = oneshot::channel();
+                if let Err(e) = to_swarm
+                    .send(swarm::ToSwarm::Publish {
+                        topic: "test-net".to_string(),
+                        data: line.as_bytes().to_vec(),
+                        result_sender: tx,
+                    })
+                    .await
+                {
+                    println!("Send error: {e:?}");
+                    return;
+                };
+                match rx.await {
+                    Ok(Err(e)) => println!("Publish error: {e:?}"),
+                    Err(e) => println!("Publish error: {e:?}"),
+                    Ok(_) => {}
+                }
+            }
+        }
+    });
+
+    // Kick it off
+    loop {
+        // on gossipsub outgoing
+        match swarm.next().await {
+            // on gossipsub incoming
+            Some(FromSwarm::Discovered { peer_id }) => {
+                println!("\n\nconnected to {peer_id}\n\n")
+            }
+            Some(FromSwarm::Expired { peer_id }) => {
+                println!("\n\ndisconnected from {peer_id}\n\n")
+            }
+            Some(FromSwarm::Message { from, topic, data }) => {
+                println!("{topic}/{from}:\n{}", String::from_utf8_lossy(&data))
+            }
+            None => {}
+        }
+    }
+}

rust/networking/examples/do_nothing.rs

@@ -1,34 +0,0 @@
-use networking;
-use tracing::{info, warn};
-use zenoh::{Result, Wait};
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    zenoh::init_log_from_env_or("info");
-    info!("Opening session...");
-    let cfg = networking::cfg(&format!("{:x}", rand::random::<u128>()), 52414)?;
-    let session = networking::open(cfg, "exo", 52414, 52413).await?;
-    let _tok = session
-        .z
-        .liveliness()
-        .declare_token(format!("nodes/{}/live", session.z.zid()))
-        .wait()?;
-    let subs = session
-        .z
-        .liveliness()
-        .declare_subscriber("**")
-        .history(true)
-        .wait()?;
-    loop {
-        tokio::select! {
-            _ = tokio::signal::ctrl_c() => break,
-            s = subs.recv_async() => {
-                match s {
-                    Err(e) => warn!("{e}"),
-                    Ok(s) => info!("{}: {}", s.kind(), s.key_expr().to_string().split("/").nth(1).unwrap()),
-                }
-            }
-        }
-    }
-    Ok(())
-}

rust/networking/examples/heartbeat.rs

@@ -1,32 +0,0 @@
-use env_logger::Env;
-use log::info;
-use networking;
-use zenoh::{Result, Wait};
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    env_logger::try_init_from_env(Env::new().default_filter_or("info")).expect("logger failed");
-    info!("Opening session...");
-    let cfg = networking::cfg(&format!("{:x}", rand::random::<u128>()), 52414)?;
-    let session = networking::open(cfg, "exo", 52414, 52413).await?;
-    let _tok = session
-        .z
-        .liveliness()
-        .declare_token(format!("nodes/{}/live", session.z.zid()))
-        .wait()?;
-    session
-        .z
-        .liveliness()
-        .declare_subscriber("**")
-        .history(true)
-        .callback(|tok| info!("{}: {}", tok.kind(), tok.key_expr().to_string()))
-        .background()
-        .wait()?;
-    loop {
-        tokio::select! {
-            _ = tokio::signal::ctrl_c() => break,
-            _ = session.z.put("hello", "world") => {},
-        }
-    }
-    Ok(())
-}

rust/networking/examples/put_string.rs

@@ -1,48 +0,0 @@
-use std::{env, time::Duration};
-
-use env_logger::Env;
-use log::info;
-use networking;
-use zenoh::Result;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    env_logger::try_init_from_env(Env::new().default_filter_or("info")).expect("logger failed");
-    let n_bytes = env::args()
-        .nth(1)
-        .and_then(|it| it.parse::<usize>().ok())
-        .expect("USAGE: put_string <n> -- pub a string of n bytes into stream/data");
-    info!("Opening session...");
-    let cfg = networking::cfg(&format!("{:x}", rand::random::<u128>()), 52414)?;
-    let session = networking::open(cfg, "exo", 52414, 52413).await?;
-    let _tok = session
-        .z
-        .liveliness()
-        .declare_token(format!("nodes/{}/live", session.z.zid()))
-        .await?;
-    let key_expr = "stream/data";
-    let payload = "n".repeat(n_bytes);
-
-    let pubs = session
-        .z
-        .declare_publisher(key_expr)
-        .congestion_control(zenoh::qos::CongestionControl::Block)
-        .await?;
-    let pubs_l = pubs.matching_listener().await?;
-    if !pubs.matching_status().await?.matching() {
-        while !pubs_l.recv_async().await?.matching() {}
-    }
-
-    tokio::time::sleep(Duration::from_secs(1)).await;
-    info!("Putting Data ('{key_expr}': '{}')...", payload.len());
-    for _ in 0..10 {
-        let t = tokio::time::Instant::now();
-        for _ in 0..5000 {
-            pubs.put(payload.clone()).await?;
-        }
-        info!("{:?}", t.elapsed());
-        tokio::time::sleep(Duration::from_secs(1)).await;
-    }
-    tokio::signal::ctrl_c().await?;
-    Ok(())
-}

rust/networking/examples/serve_storage.rs

@@ -1,74 +0,0 @@
-use std::time::Duration;
-
-use env_logger::Env;
-use log::info;
-use networking;
-use zenoh::Result;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    env_logger::try_init_from_env(Env::new().default_filter_or("info")).expect("logger failed");
-    info!("Opening session...");
-    let cfg = networking::cfg(&format!("{:x}", rand::random::<u128>()), 52414)?;
-    let session = networking::open(cfg, "exo", 52414, 52413).await?;
-    let _tok = session
-        .z
-        .liveliness()
-        .declare_token(format!("nodes/{}/live", session.z.zid()))
-        .await?;
-    let _sub = session
-        .z
-        .liveliness()
-        .declare_subscriber("nodes/*/live")
-        .history(true)
-        .callback(|tok| {
-            info!(
-                "{}: {}",
-                tok.kind(),
-                tok.key_expr()
-                    .to_string()
-                    .strip_prefix("nodes/")
-                    .and_then(|it| it.strip_suffix("/live"))
-                    .unwrap()
-            )
-        })
-        .await?;
-
-    let watch = async {
-        for _ in 0..1000 {
-            tokio::time::sleep(Duration::from_secs(1)).await;
-            session
-                .z
-                .get("**")
-                .callback(|reply| {
-                    let sample = reply.into_result().expect("no errs");
-                    info!(
-                        "got {} bytes on {}",
-                        sample.payload().len(),
-                        sample.key_expr()
-                    )
-                })
-                .await?;
-        }
-        Result::<()>::Ok(())
-    };
-    let subs = session.z.declare_subscriber("**").await?;
-
-    let mut i = 0;
-    let _a = async {
-        while let Ok(sample) = subs.recv_async().await {
-            i += 1;
-            info!(
-                "[{i}] received {} bytes on {}",
-                sample.payload().len(),
-                sample.key_expr()
-            )
-        }
-    };
-    tokio::select! {
-        _ = watch => {},
-        _ = _a => {},
-        _ = tokio::signal::ctrl_c() => {},
-    }
-    Ok(())
-}

rust/networking/examples/z_get.rs

@@ -1,41 +0,0 @@
-use std::{borrow::Cow, env};
-
-use env_logger::Env;
-use log::{info, warn};
-use networking;
-use zenoh::{Result, Wait};
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    env_logger::try_init_from_env(Env::new().default_filter_or("info")).expect("logger failed");
-    info!("Opening session...");
-    let cfg = networking::cfg(&format!("{:x}", rand::random::<u128>()), 52414)?;
-    let session = networking::open(cfg, "exo", 52414, 52413).await?;
-    let other_live = session
-        .z
-        .liveliness()
-        .declare_subscriber("**")
-        .history(true)
-        .wait()?;
-    _ = other_live.recv_async().await?;
-    let other_live = session.z.liveliness().get("**").wait()?;
-    while let Ok(s) = other_live.recv_async().await {
-        info!("{s:?}");
-    }
-    let query = env::args().nth(1).expect("USAGE: z_get [query]");
-    info!("Querying {query}");
-    let subs = session.z.liveliness().get(query).await?;
-    while let Ok(r) = subs.recv_async().await {
-        match r.into_result() {
-            Ok(s) => info!(
-                "{}: {}",
-                s.key_expr(),
-                s.payload()
-                    .try_to_string()
-                    .unwrap_or_else(|_| Cow::Borrowed("-bytes-"))
-            ),
-            Err(e) => warn!("{e}"),
-        }
-    }
-    Ok(())
-}

rust/networking/src/RESEARCH_NOTES.txt

@@ -0,0 +1,44 @@
+https://github.com/ml-explore/mlx/commit/3fe98bacc7640d857acf3539f1d21b47a32e5609
+^raw sockets distributed -> `<net/ndrv.h>` -> https://newosxbook.com/code/xnu-3247.1.106/bsd/net/ndrv.h.auto.html
+--> header file for a networking component found in the macOS kernel (XNU) that defines structures for network device driver registration, specifically the ndrv_demux_desc and ndrv_protocol_desc structures used for demultiplexing protocol data at the network interface level. It specifies how to describe protocol data, such as an Ethernet type or a SNAP header, and how to associate these descriptions with a specific protocol family to receive matching packets.
+--> Used to bind an NDRV socket so that packets that match given protocol demux descriptions can be received.
+--> An NDRV socket is a special kind of socket in the Darwin/macOS operating system's XNU kernel, used for low-level network packet manipulation and binding to specific protocols for packet processing. It allows user-space applications or drivers to directly write Layer 2 (L2) network packets or interact with the network stack at a lower level, often by binding to protocol descriptors like the ndrv_protocol_desc. This type of socket is used for functions such as capturing and injecting packets, especially in network infrastructure software like routers or for kernel-level network monitoring and security tools.
+--> also called PF_NDRV sockets --> https://newosxbook.com/bonus/vol1ch16.html
+----> they are conceptually similar to https://scapy.disruptivelabs.in/networking/socket-interface PF_RAW or PF_PACKET
+
+https://stackoverflow.com/questions/17169298/af-packet-on-osx
+^AF_PACKET duplicates the packets as soon as it receives them from the physical layer (for incoming packets) or just before sending them out to the physical layer (for outgoing packets). -> this is on Linux only
+^it doesn't exist on OS X so you can use /dev/bpfX (Berkeley Packet Filter) for sniffing
+
+https://www.unix.com/man_page/mojave/4/ip/
+^OS X manpages for IP
+
+https://developer.apple.com/documentation/kernel/implementing_drivers_system_extensions_and_kexts
+^driver kit, system extensions & kexts for macOS
+
+----
+
+To set up a Linux system to use a Thunderbolt connection as a network device, connect the two computers with a Thunderbolt cable, load the thunderbolt-net kernel module (usually automatic but modprobe is an option for manual loading), and then the operating system will create virtual Ethernet interfaces (e.g., thunderbolt0) for networking. You can then use standard tools like ifconfig or your desktop environment's network manager to configure these new interfaces for a link-local network.
+--> https://gist.github.com/geosp/80fbd39e617b7d1d9421683df4ea224a
+----> here is a guide on how to set up thunderbolt-ethernet on linux
+----> I may be able to steal the thunderbolt-net code ideas to implement a kernel module for MacOS
+
+https://chatgpt.com/s/t_68af8e41a8548191993281a014f846a7
+^GPT discussion about making socket interface
+
+https://chatgpt.com/s/t_68afb798a85c8191973c02a0fa7a48a3 --> link-local address,,??
+https://chatgpt.com/s/t_68afb02987e08191b2b0044d3667ece2
+^GPT discussion about accessing TB on MacOS low level interactions
+
+--------------------------------
+
+https://www.intel.com/content/www/us/en/support/articles/000098893/software.html
+^Thunderbolt Share & Thunderbolt Networking Mode => intel's equivalent of thunderbolt bridge
+
+
+---------------------------------
+
+https://www.zerotier.com/blog/how-zerotier-eliminated-kernel-extensions-on-macos/
+-->fake ethernet devices on MacOS -> omg??? we can detect thunderbolt bridge, then bind to it, then re-expose it as fake ethernet??
+-->ps: https://chatgpt.com/s/t_68afb2b25fb881919526763fb5d7359c, AF/PF_NDRV are one and the same!!!
+-->https://github.com/zerotier/ZeroTierOne/blob/dev/osdep/MacEthernetTapAgent.c

rust/networking/src/discovery.rs

@@ -1,339 +1,390 @@
-use std::{
-    io,
-    net::{Ipv6Addr, SocketAddr, SocketAddrV6},
-    sync::Arc,
-    time::Duration,
+use crate::ext::MultiaddrExt;
+use delegate::delegate;
+use either::Either;
+use futures_lite::FutureExt;
+use futures_timer::Delay;
+use libp2p::core::transport::PortUse;
+use libp2p::core::{ConnectedPoint, Endpoint};
+use libp2p::swarm::behaviour::ConnectionEstablished;
+use libp2p::swarm::dial_opts::DialOpts;
+use libp2p::swarm::{
+    CloseConnection, ConnectionClosed, ConnectionDenied, ConnectionHandler,
+    ConnectionHandlerSelect, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent,
+    THandlerOutEvent, ToSwarm, dummy,
 };
+use libp2p::{Multiaddr, PeerId, identity, mdns};
+use std::collections::{BTreeSet, HashMap};
+use std::convert::Infallible;
+use std::io;
+use std::net::IpAddr;
+use std::task::{Context, Poll};
+use std::time::Duration;
+use util::wakerdeque::WakerDeque;
 
-use bytemuck::{Pod, Zeroable};
-use log::{debug, trace, warn};
-use netwatcher::WatchHandle;
-use parking_lot::Mutex;
-use tokio::{
-    net::UdpSocket,
-    time::{Interval, interval},
-};
-use zenoh::config::ZenohId;
+const RETRY_CONNECT_INTERVAL: Duration = Duration::from_secs(5);
 
-const GROUP: Ipv6Addr = Ipv6Addr::new(0xff12, 0, 0, 0, 0, 0, 0xe0a1, 0xde89);
-const MAGIC: [u8; 3] = *b"EXO";
+mod managed {
+    use libp2p::swarm::NetworkBehaviour;
+    use libp2p::{identity, mdns, ping};
+    use std::io;
+    use std::time::Duration;
 
-pub struct Discovery {
-    sock: Arc<UdpSocket>,
-    ifaces: Arc<Mutex<Vec<SocketAddrV6>>>,
-    namespace: [u8; 8],
-    last_nonce: Mutex<[u8; 8]>,
-    /// the port of the service we are doing discovery for - transmitted to peers
-    listen_port: u16,
-    zid: ZenohId,
-    tick: Interval,
-    _sync: Mutex<WatchHandle>,
-}
+    const MDNS_RECORD_TTL: Duration = Duration::from_secs(2_500);
+    const MDNS_QUERY_INTERVAL: Duration = Duration::from_secs(1_500);
+    const PING_TIMEOUT: Duration = Duration::from_millis(2_500);
+    const PING_INTERVAL: Duration = Duration::from_millis(2_500);
 
-#[derive(Debug, Clone, Copy)]
-pub struct Discovered {
-    pub zid: ZenohId,
-    pub addr: SocketAddrV6,
-}
+    #[derive(NetworkBehaviour)]
+    pub struct Behaviour {
+        mdns: mdns::tokio::Behaviour,
+        ping: ping::Behaviour,
+    }
 
-impl Discovery {
-    pub async fn new(
-        zid: ZenohId,
-        namespace: [u8; 8],
-        listen_port: u16,
-        discovery_port: u16,
-    ) -> io::Result<Self> {
-        let sock = socket2::Socket::new(
-            socket2::Domain::IPV6,
-            socket2::Type::DGRAM,
-            Some(socket2::Protocol::UDP),
-        )?;
-        sock.set_reuse_address(true)?;
-        #[cfg(unix)]
-        sock.set_reuse_port(true)?;
-        sock.bind(&SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, discovery_port, 0, 0).into())?;
-        sock.set_nonblocking(true)?;
-        sock.set_multicast_loop_v6(true)?;
-        let sock = Arc::new(UdpSocket::from_std(sock.into())?);
-        let ifaces: Arc<Mutex<Vec<SocketAddrV6>>> = Default::default();
-        let _sync = Mutex::new(
-            netwatcher::watch_interfaces_with_callback({
-                let sock = sock.clone();
-                let ifaces = ifaces.clone();
-                move |update| {
-                    for (iface_idx, iface) in update.interfaces.iter() {
-                        if iface
-                            .ipv6_ips()
-                            .all(|addr| addr.is_loopback() || addr.is_unspecified())
-                        {
-                            continue;
-                        }
-
-                        match sock.join_multicast_v6(&GROUP, *iface_idx) {
-                            Ok(()) => ifaces.lock().push(SocketAddrV6::new(
-                                GROUP,
-                                discovery_port,
-                                0,
-                                *iface_idx,
-                            )),
-                            Err(e) if e.kind() != io::ErrorKind::AddrInUse => {
-                                // skip AddrInUse - just means we've already joined the mv6
-                                if let Some(iface) = update.interfaces.get(&iface_idx) {
-                                    warn!(
-                                        "failed to join multicast v6 for interface {}: {e}",
-                                        iface.name
-                                    )
-                                }
-                            }
-                            _ => {}
-                        }
-                    }
-                    for iface_idx in update.diff.removed {
-                        ifaces.lock().retain(|addr| addr.scope_id() != iface_idx);
-
-                        if let Err(e) = sock.leave_multicast_v6(&GROUP, iface_idx) {
-                            if let Some(iface) = update.interfaces.get(&iface_idx) {
-                                warn!(
-                                    "failed to leave multicast v6 for interface {}: {e}",
-                                    iface.name
-                                )
-                            }
-                        }
-                    }
-                }
+    impl Behaviour {
+        pub fn new(keypair: &identity::Keypair) -> io::Result<Self> {
+            Ok(Self {
+                mdns: mdns_behaviour(keypair)?,
+                ping: ping_behaviour(),
             })
-            // todo: better error handling here
-            .expect("failed to bind discovery watcher"),
-        );
+        }
+    }
+
+    fn mdns_behaviour(keypair: &identity::Keypair) -> io::Result<mdns::tokio::Behaviour> {
+        use mdns::{Config, tokio};
+
+        // mDNS config => enable IPv6
+        let mdns_config = Config {
+            ttl: MDNS_RECORD_TTL,
+            query_interval: MDNS_QUERY_INTERVAL,
+
+            // enable_ipv6: true, // TODO: for some reason, TCP+mDNS don't work well with ipv6?? figure out how to make work
+            ..Default::default()
+        };
+
+        let mdns_behaviour = tokio::Behaviour::new(mdns_config, keypair.public().to_peer_id());
+        Ok(mdns_behaviour?)
+    }
+
+    fn ping_behaviour() -> ping::Behaviour {
+        ping::Behaviour::new(
+            ping::Config::new()
+                .with_timeout(PING_TIMEOUT)
+                .with_interval(PING_INTERVAL),
+        )
+    }
+}
+
+/// Events for when a listening connection is truly established and truly closed.
+#[derive(Debug, Clone)]
+pub enum Event {
+    ConnectionEstablished {
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+        remote_ip: IpAddr,
+        remote_tcp_port: u16,
+    },
+    ConnectionClosed {
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+        remote_ip: IpAddr,
+        remote_tcp_port: u16,
+    },
+}
+
+/// Discovery behavior that wraps mDNS to produce truly discovered durable peer-connections.
+///
+/// The behaviour operates as such:
+///  1) All true (listening) connections/disconnections are tracked, emitting corresponding events
+///     to the swarm.
+///  1) mDNS discovered/expired peers are tracked; discovered but not connected peers are dialed
+///     immediately, and expired but connected peers are disconnected from immediately.
+///  2) Every fixed interval: discovered but not connected peers are dialed, and expired but
+///     connected peers are disconnected from.
+pub struct Behaviour {
+    // state-tracking for managed behaviors & mDNS-discovered peers
+    managed: managed::Behaviour,
+    mdns_discovered: HashMap<PeerId, BTreeSet<Multiaddr>>,
+    bootstrap_peers: Vec<Multiaddr>,
+
+    retry_delay: Delay, // retry interval
+
+    // pending events to emmit => waker-backed Deque to control polling
+    pending_events: WakerDeque<ToSwarm<Event, Infallible>>,
+}
+
+impl Behaviour {
+    pub fn new(keypair: &identity::Keypair, bootstrap_peers: Vec<Multiaddr>) -> io::Result<Self> {
         Ok(Self {
-            sock,
-            namespace,
-            ifaces,
-            last_nonce: Mutex::new(rand::random()),
-            listen_port,
-            zid,
-            tick: interval(Duration::from_secs(1)),
-            _sync,
+            managed: managed::Behaviour::new(keypair)?,
+            mdns_discovered: HashMap::new(),
+            bootstrap_peers,
+            retry_delay: Delay::new(RETRY_CONNECT_INTERVAL),
+            pending_events: WakerDeque::new(),
         })
     }
 
-    pub async fn next(&mut self) -> io::Result<Discovered> {
-        let mut buf = [0u8; Hello::buf_size() + WhatsUp::buf_size() + 1];
-        loop {
-            tokio::select! {
-                _ = self.tick.tick() => {
-                    self.announce().await?;
+    fn dial(&mut self, peer_id: PeerId, addr: Multiaddr) {
+        self.pending_events.push_back(ToSwarm::Dial {
+            opts: DialOpts::peer_id(peer_id).addresses(vec![addr]).build(),
+        })
+    }
+
+    fn close_connection(&mut self, peer_id: PeerId, connection: ConnectionId) {
+        // push front to make this IMMEDIATE
+        self.pending_events.push_front(ToSwarm::CloseConnection {
+            peer_id,
+            connection: CloseConnection::One(connection),
+        })
+    }
+
+    fn handle_mdns_discovered(&mut self, peers: Vec<(PeerId, Multiaddr)>) {
+        for (p, ma) in peers {
+            self.dial(p, ma.clone()); // always connect
+
+            // get peer's multi-addresses or insert if missing
+            let Some(mas) = self.mdns_discovered.get_mut(&p) else {
+                self.mdns_discovered.insert(p, BTreeSet::from([ma]));
+                continue;
+            };
+
+            // multiaddress should never already be present - else something has gone wrong
+            let is_new_addr = mas.insert(ma);
+            assert!(is_new_addr, "cannot discover a discovered peer");
+        }
+    }
+
+    fn handle_mdns_expired(&mut self, peers: Vec<(PeerId, Multiaddr)>) {
+        for (p, ma) in peers {
+            // at this point, we *must* have the peer
+            let mas = self
+                .mdns_discovered
+                .get_mut(&p)
+                .expect("nonexistent peer cannot expire");
+
+            // at this point, we *must* have the multiaddress
+            let was_present = mas.remove(&ma);
+            assert!(was_present, "nonexistent multiaddress cannot expire");
+
+            // if empty, remove the peer-id entirely
+            if mas.is_empty() {
+                self.mdns_discovered.remove(&p);
+            }
+        }
+    }
+
+    fn on_connection_established(
+        &mut self,
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+        remote_ip: IpAddr,
+        remote_tcp_port: u16,
+    ) {
+        // send out connected event
+        self.pending_events
+            .push_back(ToSwarm::GenerateEvent(Event::ConnectionEstablished {
+                peer_id,
+                connection_id,
+                remote_ip,
+                remote_tcp_port,
+            }));
+    }
+
+    fn on_connection_closed(
+        &mut self,
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+        remote_ip: IpAddr,
+        remote_tcp_port: u16,
+    ) {
+        // send out disconnected event
+        self.pending_events
+            .push_back(ToSwarm::GenerateEvent(Event::ConnectionClosed {
+                peer_id,
+                connection_id,
+                remote_ip,
+                remote_tcp_port,
+            }));
+    }
+}
+
+impl NetworkBehaviour for Behaviour {
+    type ConnectionHandler =
+        ConnectionHandlerSelect<dummy::ConnectionHandler, THandler<managed::Behaviour>>;
+    type ToSwarm = Event;
+
+    // simply delegate to underlying mDNS behaviour
+
+    delegate! {
+        to self.managed {
+            fn handle_pending_inbound_connection(&mut self, connection_id: ConnectionId, local_addr: &Multiaddr, remote_addr: &Multiaddr) -> Result<(), ConnectionDenied>;
+            fn handle_pending_outbound_connection(&mut self, connection_id: ConnectionId, maybe_peer: Option<PeerId>, addresses: &[Multiaddr], effective_role: Endpoint) -> Result<Vec<Multiaddr>, ConnectionDenied>;
+        }
+    }
+
+    fn handle_established_inbound_connection(
+        &mut self,
+        connection_id: ConnectionId,
+        peer: PeerId,
+        local_addr: &Multiaddr,
+        remote_addr: &Multiaddr,
+    ) -> Result<THandler<Self>, ConnectionDenied> {
+        Ok(ConnectionHandler::select(
+            dummy::ConnectionHandler,
+            self.managed.handle_established_inbound_connection(
+                connection_id,
+                peer,
+                local_addr,
+                remote_addr,
+            )?,
+        ))
+    }
+
+    #[allow(clippy::needless_question_mark)]
+    fn handle_established_outbound_connection(
+        &mut self,
+        connection_id: ConnectionId,
+        peer: PeerId,
+        addr: &Multiaddr,
+        role_override: Endpoint,
+        port_use: PortUse,
+    ) -> Result<THandler<Self>, ConnectionDenied> {
+        Ok(ConnectionHandler::select(
+            dummy::ConnectionHandler,
+            self.managed.handle_established_outbound_connection(
+                connection_id,
+                peer,
+                addr,
+                role_override,
+                port_use,
+            )?,
+        ))
+    }
+
+    fn on_connection_handler_event(
+        &mut self,
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+        event: THandlerOutEvent<Self>,
+    ) {
+        match event {
+            Either::Left(ev) => libp2p::core::util::unreachable(ev),
+            Either::Right(ev) => {
+                self.managed
+                    .on_connection_handler_event(peer_id, connection_id, ev)
+            }
+        }
+    }
+
+    // hook into these methods to drive behavior
+
+    fn on_swarm_event(&mut self, event: FromSwarm) {
+        self.managed.on_swarm_event(event); // let mDNS handle swarm events
+
+        // handle swarm events to update internal state:
+        match event {
+            FromSwarm::ConnectionEstablished(ConnectionEstablished {
+                peer_id,
+                connection_id,
+                endpoint,
+                ..
+            }) => {
+                let remote_address = match endpoint {
+                    ConnectedPoint::Dialer { address, .. } => address,
+                    ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr,
+                };
+
+                if let Some((ip, port)) = remote_address.try_to_tcp_addr() {
+                    // handle connection established event which is filtered correctly
+                    self.on_connection_established(peer_id, connection_id, ip, port)
                 }
-                res = self.sock.recv_from(&mut buf) => {
-                    let Ok((bytes_read, addr)) = res else { continue; };
-                    if let Some(discovered) = self.respond(bytes_read, addr, &buf).await? {
-                        return Ok(discovered)
+            }
+            FromSwarm::ConnectionClosed(ConnectionClosed {
+                peer_id,
+                connection_id,
+                endpoint,
+                ..
+            }) => {
+                let remote_address = match endpoint {
+                    ConnectedPoint::Dialer { address, .. } => address,
+                    ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr,
+                };
+
+                if let Some((ip, port)) = remote_address.try_to_tcp_addr() {
+                    // handle connection closed event which is filtered correctly
+                    self.on_connection_closed(peer_id, connection_id, ip, port)
+                }
+            }
+
+            // since we are running TCP/IP transport layer, we are assuming that
+            // no address changes can occur, hence encountering one is a fatal error
+            FromSwarm::AddressChange(a) => {
+                unreachable!("unhandlable: address change encountered: {:?}", a)
+            }
+            _ => {}
+        }
+    }
+
+    fn poll(&mut self, cx: &mut Context) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
+        // delegate to managed behaviors for any behaviors they need to perform
+        match self.managed.poll(cx) {
+            Poll::Ready(ToSwarm::GenerateEvent(e)) => {
+                match e {
+                    // handle discovered and expired events from mDNS
+                    managed::BehaviourEvent::Mdns(e) => match e.clone() {
+                        mdns::Event::Discovered(peers) => {
+                            self.handle_mdns_discovered(peers);
+                        }
+                        mdns::Event::Expired(peers) => {
+                            self.handle_mdns_expired(peers);
+                        }
+                    },
+
+                    // handle ping events => if error then disconnect
+                    managed::BehaviourEvent::Ping(e) => {
+                        if let Err(_) = e.result {
+                            self.close_connection(e.peer, e.connection.clone())
+                        }
                     }
                 }
+
+                // since we just consumed an event, we should immediately wake just in case
+                // there are more events to come where that came from
+                cx.waker().wake_by_ref();
             }
-        }
-    }
 
-    async fn respond(
-        &self,
-        bytes_read: usize,
-        addr: SocketAddr,
-        buf: &[u8],
-    ) -> io::Result<Option<Discovered>> {
-        trace!(
-            "raw recv: {bytes_read} bytes from {addr}: {:02x?}",
-            &buf[..bytes_read]
-        );
-        if bytes_read < size_of::<Header>() {
-            trace!("dropped: early EOF");
-            return Ok(None);
-        }
-        let header: &Header = bytemuck::from_bytes(&buf[0..size_of::<Header>()]);
-        if header.magic != MAGIC {
-            trace!("dropped: wrong magic");
-            return Ok(None);
-        }
-        let Ok(kind) = header.kind.try_into() else {
-            trace!("dropped: unknown message kind {}", header.kind);
-            return Ok(None);
-        };
-        match kind {
-            Kind::Hello => {
-                let total = Hello::buf_size();
-                if bytes_read != total {
-                    trace!("dropped: hello wrong size");
-                    return Ok(None);
-                }
-                let hello: &Hello = bytemuck::from_bytes(&buf[size_of::<Header>()..total]);
-                if hello.nonce == *self.last_nonce.lock() {
-                    trace!("dropped: local hello nonce");
-                    return Ok(None);
-                }
-                if hello.namespace != self.namespace {
-                    trace!("dropped: different namespace");
-                    return Ok(None);
-                }
-
-                // reply
-                trace!("replying to Hello({:?})", hello.nonce);
-                let reply = WhatsUp {
-                    nonce: hello.nonce,
-                    zid: self.zid.to_le_bytes(),
-                    port_le: self.listen_port.to_le_bytes(),
-                }
-                .alloc();
-
-                for i in 1..6 {
-                    if self
-                        .sock
-                        .send_to(&reply, addr)
-                        .await
-                        .inspect_err(|e| debug!("send to {addr} failed: {e}"))
-                        .is_ok_and(|sent| sent == WhatsUp::buf_size())
-                    {
-                        trace!(
-                            "sent {} bytes to {addr} after {} attempt(s)",
-                            WhatsUp::buf_size(),
-                            i
-                        );
-                        break;
-                    }
-                    tokio::time::sleep(Duration::from_millis(300)).await;
-                }
-                Ok(None)
+            // forward any other mDNS event to the swarm or its connection handler(s)
+            Poll::Ready(e) => {
+                return Poll::Ready(
+                    e.map_out(|_| unreachable!("events returning to swarm already handled"))
+                        .map_in(Either::Right),
+                );
             }
-            Kind::WhatsUp => {
-                let total = WhatsUp::buf_size();
-                if bytes_read != total {
-                    trace!("dropped: whatsup wrong size");
-                    return Ok(None);
-                }
-                let whats_up: &WhatsUp = bytemuck::from_bytes(&buf[size_of::<Header>()..total]);
-                if whats_up.nonce != *self.last_nonce.lock() {
-                    trace!("dropped: stale nonce");
-                    return Ok(None);
-                }
-                let SocketAddr::V6(v6) = addr else {
-                    trace!("dropped: v4 addr used");
-                    return Ok(None);
-                };
-                let Ok(zid) = ZenohId::try_from(&whats_up.zid[..]) else {
-                    trace!("dropped: zenoh conversion failed");
-                    return Ok(None);
-                };
-                if zid == self.zid {
-                    trace!("dropped: self zenoh id");
-                    return Ok(None);
-                }
-                // discovery success!
-                // the incoming port is our listen port;
-                // overwrite it with the whats_up port corresponding to the remote zenoh service
-                let addr = {
-                    let mut x = v6;
-                    x.set_port(u16::from_le_bytes(whats_up.port_le));
-                    x
-                };
-                Ok(Some(Discovered { addr, zid }))
-            }
-        }
-    }
 
-    async fn announce(&self) -> io::Result<()> {
-        let nonce = rand::random();
-        *self.last_nonce.lock() = nonce;
-        let buf = Hello {
-            nonce,
-            namespace: self.namespace,
+            Poll::Pending => {}
         }
-        .alloc();
 
-        let addrs = self.ifaces.lock().clone();
-        debug!("announcing Hello({nonce:?}) to {addrs:?}");
-        // rev so .remove() doesn't break things
-        for (i, addr) in addrs.into_iter().enumerate().rev() {
-            match self.sock.send_to(&buf, addr).await {
-                Ok(bytes) => trace!("sent {bytes} to {addr}"),
-                Err(e) if e.kind() == io::ErrorKind::HostUnreachable => {
-                    debug!("disabling discovery address {addr}: {e}");
-                    _ = self.ifaces.lock().swap_remove(i);
+        // retry connecting to all mDNS peers periodically (fails safely if already connected)
+        if self.retry_delay.poll(cx).is_ready() {
+            for (p, mas) in self.mdns_discovered.clone() {
+                for ma in mas {
+                    self.dial(p, ma)
                 }
-                Err(e) => debug!("failed to reach {addr}: {e}"),
             }
+            // dial bootstrap peers (for environments where mDNS is unavailable)
+            for addr in &self.bootstrap_peers {
+                self.pending_events.push_back(ToSwarm::Dial {
+                    opts: DialOpts::unknown_peer_id().address(addr.clone()).build(),
+                })
+            }
+            self.retry_delay.reset(RETRY_CONNECT_INTERVAL) // reset timeout
         }
-        Ok(())
+
+        // send out any pending events from our own service
+        if let Some(e) = self.pending_events.pop_front(cx) {
+            return Poll::Ready(e.map_in(Either::Left));
+        }
+
+        // wait for pending events
+        Poll::Pending
     }
 }
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy)]
-// packet & version
-pub enum Kind {
-    Hello = 0,
-    WhatsUp = 1,
-}
-
-pub struct UnknownKind;
-impl TryFrom<u8> for Kind {
-    type Error = UnknownKind;
-    fn try_from(value: u8) -> Result<Self, Self::Error> {
-        match value {
-            0 => Ok(Self::Hello),
-            1 => Ok(Self::WhatsUp),
-            _ => Err(UnknownKind),
-        }
-    }
-}
-
-pub trait Message: Pod {
-    const KIND: Kind;
-}
-// should be part of the Message trait, but const in traits isnt stabilized. this lets alloc :: Self -> [u8; Self::buf_size()]
-macro_rules! impl_alloc {
-    ($a:ident) => {
-        impl $a {
-            const fn buf_size() -> usize {
-                size_of::<Header>() + size_of::<Self>()
-            }
-            pub fn alloc(self) -> [u8; Self::buf_size()] {
-                let mut buf = [0u8; Self::buf_size()];
-                buf[0..size_of::<Header>()].copy_from_slice(bytemuck::bytes_of(&Header {
-                    magic: MAGIC,
-                    kind: Self::KIND as u8,
-                }));
-                buf[size_of::<Header>()..Self::buf_size()]
-                    .copy_from_slice(bytemuck::bytes_of(&self));
-                buf
-            }
-        }
-    };
-}
-
-#[repr(C)]
-#[derive(Debug, Clone, Copy, Pod, Zeroable)]
-pub struct Header {
-    magic: [u8; 3],
-    kind: u8,
-}
-
-#[repr(C)]
-#[derive(Debug, Clone, Copy, Pod, Zeroable)]
-pub struct Hello {
-    pub nonce: [u8; 8],
-    pub namespace: [u8; 8],
-}
-impl Message for Hello {
-    const KIND: Kind = Kind::Hello;
-}
-impl_alloc!(Hello);
-
-#[repr(C)]
-#[derive(Debug, Clone, Copy, Pod, Zeroable)]
-pub struct WhatsUp {
-    pub nonce: [u8; 8],
-    pub zid: [u8; 16],
-    pub port_le: [u8; 2],
-}
-impl Message for WhatsUp {
-    const KIND: Kind = Kind::WhatsUp;
-}
-impl_alloc!(WhatsUp);

rust/networking/src/lib.rs

@@ -1,116 +1,44 @@
-use std::sync::Arc;
-
-use tokio::task::JoinHandle;
-use zenoh::{Result, Session as ZSession, config::Locator};
-use zenoh_plugin_storage_manager::StoragesPlugin;
-use zenoh_plugin_trait::PluginsManager;
-
-pub use zenoh::{Config, config::ZenohId};
-
-use crate::discovery::Discovery;
-
+//! TODO: crate documentation
+//!
+//! this is here as a placeholder documentation
+//!
+//!
 pub mod discovery;
 pub mod swarm;
 
-pub fn is_valid_zid(identity: &str) -> bool {
-    let mut iter = identity.chars();
-    iter.next()
-        .is_some_and(|c| ('1'..='9').contains(&c) || ('a'..='f').contains(&c))
-        && iter.all(|c| ('0'..='9').contains(&c) || ('a'..='f').contains(&c))
-        && identity.len() <= 32
+/// Namespace for all the type/trait aliases used by this crate.
+pub(crate) mod alias {
+    use std::error::Error;
+
+    pub type AnyError = Box<dyn Error + Send + Sync + 'static>;
+    pub type AnyResult<T> = Result<T, AnyError>;
 }
 
-pub fn cfg(identity: &str, listen_port: u16) -> Result<zenoh::Config> {
-    assert!(is_valid_zid(identity));
-    assert!(identity.len() <= 32);
-    assert!(listen_port != 0, "must used defined listen port");
-    let mut cfg = zenoh::Config::default();
-    // todo: cleanup
-    cfg.insert_json5("id", &format!("\"{identity}\""))?;
-    cfg.insert_json5("mode", "\"router\"")?;
-    cfg.insert_json5("listen/endpoints", &format!("[\"tcp/[::]:{listen_port}\"]"))?;
-    cfg.insert_json5("scouting/multicast/enabled", "false")?;
-    cfg.insert_json5("scouting/multicast/autoconnect", "[]")?;
-    cfg.insert_json5("scouting/gossip/multihop", "true")?;
-    cfg.insert_json5("adminspace/enabled", "true")?;
-    //cfg.insert_json5("transport/link/tx/batch_size", "9216")?;
-    cfg.insert_json5("transport/link/rx/buffer_size", "16777216")?;
-    //cfg.insert_json5("timestamping/enabled", "true")?;
-    cfg.insert_json5("plugins/storage_manager/__required__", "true")?;
-    cfg.insert_json5(
-        "plugins/storage_manager/storages/mem1",
-        r#"{
-            key_expr: "storage/mem1/**",
-            strip_prefix: "storage/mem1",
-            volume: "memory",
-            replication: {
-                interval: 2,
-            }
-        }"#,
-    )?;
-    Ok(cfg)
-}
+/// Namespace for crate-wide extension traits/methods
+pub(crate) mod ext {
+    use extend::ext;
+    use libp2p::Multiaddr;
+    use libp2p::multiaddr::Protocol;
+    use std::net::IpAddr;
 
-pub async fn open(
-    cfg: zenoh::Config,
-    namespace: &str,
-    listen_port: u16,
-    discovery_service_port: u16,
-) -> Result<Session> {
-    assert!(listen_port != 0, "must used defined listen port");
-    let namespace: [u8; 8] = {
-        blake3::hash(namespace.as_bytes()).as_bytes()[..8]
-            .try_into()
-            .expect("8 is equal to 8")
-    };
-    let mut plugins = PluginsManager::static_plugins_only();
-    plugins.declare_static_plugin::<StoragesPlugin, _>("storage_manager", true);
-    let mut runtime = zenoh::internal::runtime::RuntimeBuilder::new(cfg)
-        .plugins_manager(plugins)
-        .build()
-        .await?;
-    let z = zenoh::session::init(runtime.clone().into()).await?;
-    runtime.start().await?;
-    let mut discovery =
-        Discovery::new(z.zid(), namespace, listen_port, discovery_service_port).await?;
-    let _jh = Arc::new(AbortOnDrop(tokio::task::spawn(async move {
-        loop {
-            let Ok(discovered) = discovery.next().await.inspect_err(|e| {
-                log::warn!("discovery error {e}");
-            }) else {
-                continue;
+    #[ext(pub, name = MultiaddrExt)]
+    impl Multiaddr {
+        /// If the multiaddress corresponds to a TCP address, extracts it
+        fn try_to_tcp_addr(&self) -> Option<(IpAddr, u16)> {
+            let mut ps = self.into_iter();
+            let ip = if let Some(p) = ps.next() {
+                match p {
+                    Protocol::Ip4(ip) => IpAddr::V4(ip),
+                    Protocol::Ip6(ip) => IpAddr::V6(ip),
+                    _ => return None,
+                }
+            } else {
+                return None;
             };
-
-            if discovered.zid > runtime.zid() {
-                log::debug!("not connecting to peer with greater zid");
-                continue;
-            }
-
-            let Ok(locator) =
-                Locator::new("tcp", discovered.addr.to_string(), "").inspect_err(|e| {
-                    log::warn!("failed to parse locator from addr: {e}");
-                })
-            else {
-                continue;
+            let Some(Protocol::Tcp(port)) = ps.next() else {
+                return None;
             };
-
-            runtime
-                .connect_peer(&discovered.zid.into(), &[locator])
-                .await;
+            Some((ip, port))
         }
-    })));
-    Ok(Session { z, _jh })
-}
-
-struct AbortOnDrop(JoinHandle<()>);
-impl Drop for AbortOnDrop {
-    fn drop(&mut self) {
-        self.0.abort();
     }
 }
-
-#[derive(Clone)]
-pub struct Session {
-    pub z: ZSession,
-    _jh: Arc<AbortOnDrop>,
-}

rust/networking/src/swarm.rs

@@ -1,22 +1,24 @@
-//! Compat shim for the old libp2p code
-
-use std::collections::HashMap;
 use std::pin::Pin;
 
-use futures_lite::Stream;
-use tokio::sync::mpsc;
-use tokio::sync::oneshot;
-use zenoh::Result;
-use zenoh::Session;
-use zenoh::handlers::FifoChannelHandler;
-use zenoh::liveliness::LivelinessToken;
-use zenoh::pubsub::Publisher;
-use zenoh::pubsub::Subscriber;
-use zenoh::qos::CongestionControl;
-use zenoh::sample::Sample;
-use zenoh::sample::SampleKind;
+use crate::swarm::transport::tcp_transport;
+use crate::{alias, discovery};
+pub use behaviour::{Behaviour, BehaviourEvent};
+use futures_lite::{Stream, StreamExt};
+use libp2p::{PeerId, SwarmBuilder, gossipsub, identity, swarm::SwarmEvent};
+use tokio::sync::{mpsc, oneshot};
 
-#[derive(Debug)]
+/// The current version of the network: this prevents devices running different versions of the
+/// software from interacting with each other.
+///
+/// TODO: right now this is a hardcoded constant; figure out what the versioning semantics should
+///       even be, and how to inject the right version into this config/initialization. E.g. should
+///       this be passed in as a parameter? What about rapidly changing versions in debug builds?
+///       this is all VERY very hard to figure out and needs to be mulled over as a team.
+pub const NETWORK_VERSION: &[u8] = b"v0.0.1";
+pub const OVERRIDE_VERSION_ENV_VAR: &str = "EXO_LIBP2P_NAMESPACE";
+
+// Uses oneshot senders to emulate function calling apis while avoiding requiring unique ownership
+// of the Swarm.
 pub enum ToSwarm {
     Unsubscribe {
         topic: String,
@@ -24,66 +26,52 @@ pub enum ToSwarm {
     },
     Subscribe {
         topic: String,
-        result_sender: oneshot::Sender<Result<bool>>,
+        result_sender: oneshot::Sender<Result<bool, gossipsub::SubscriptionError>>,
     },
     Publish {
         topic: String,
         data: Vec<u8>,
-        result_sender: oneshot::Sender<Result<()>>,
+        result_sender: oneshot::Sender<Result<gossipsub::MessageId, gossipsub::PublishError>>,
     },
 }
-#[derive(Debug)]
 pub enum FromSwarm {
-    Message { topic: String, data: Vec<u8> },
-    Discovered {},
-    Expired {},
+    Message {
+        from: PeerId,
+        topic: String,
+        data: Vec<u8>,
+    },
+    Discovered {
+        peer_id: PeerId,
+    },
+    Expired {
+        peer_id: PeerId,
+    },
 }
 
-pub type Topics = HashMap<String, (Subscriber<()>, Publisher<'static>)>;
 pub struct Swarm {
-    pub session: crate::Session,
-    pub from_client: mpsc::Receiver<ToSwarm>,
+    swarm: libp2p::Swarm<Behaviour>,
+    from_client: mpsc::Receiver<ToSwarm>,
 }
 
 impl Swarm {
     pub fn into_stream(self) -> Pin<Box<dyn Stream<Item = FromSwarm> + Send>> {
         let Swarm {
-            session,
+            mut swarm,
             mut from_client,
         } = self;
         let stream = async_stream::stream! {
-            let mut session = session;
-            let (mut to_topics, mut from_topics) = mpsc::channel(1024);
-            let mut topics = Topics::new();
-            let Ok((_token, discovery)) = register_liveness(&mut session.z).await else { return; };
             loop {
                 tokio::select! {
                     msg = from_client.recv() => {
                         let Some(msg) = msg else { break };
-                        on_message(&mut session.z, &mut topics, &mut to_topics, msg).await;
+                        on_message(&mut swarm, msg);
                     }
-                    event = from_topics.recv() => {
-                        if let Some(event) = event {
-                            yield event
+                    event = swarm.next() => {
+                        let Some(event) = event else { break };
+                        if let Some(item) = filter_swarm_event(event) {
+                            yield item;
                         }
                     }
-                    token = discovery.recv_async() => {
-                        if let Ok(token) = token {
-                            let key_expr = token.key_expr().as_str().to_owned();
-                            let zid = key_expr.strip_prefix("live/");
-                            yield match token.kind() {
-                                SampleKind::Put => {
-                                    log::info!("discovered: {zid:?}");
-                                    FromSwarm::Discovered {}
-                                }
-                                SampleKind::Delete => {
-                                    log::info!("expired: {zid:?}");
-                                    FromSwarm::Expired {}
-                                }
-                            }
-                        }
-
-                    }
                 }
             }
         };
@@ -91,118 +79,208 @@ impl Swarm {
     }
 }
 
-async fn register_liveness(
-    session: &mut Session,
-) -> Result<(LivelinessToken, Subscriber<FifoChannelHandler<Sample>>)> {
-    let token = session
-        .liveliness()
-        .declare_token(format!("live/{}", session.zid()))
-        .await?;
-    let sub = session
-        .liveliness()
-        .declare_subscriber("live/*")
-        .history(true)
-        .await?;
-    Ok((token, sub))
-}
-
-async fn on_message(
-    session: &mut Session,
-    topics: &mut Topics,
-    to_topics: &mut mpsc::Sender<FromSwarm>,
-    msg: ToSwarm,
-) {
-    match msg {
-        ToSwarm::Publish {
+fn on_message(swarm: &mut libp2p::Swarm<Behaviour>, message: ToSwarm) {
+    match message {
+        ToSwarm::Subscribe {
             topic,
-            data,
             result_sender,
         } => {
-            let res = match topics.get(&topic) {
-                Some(topic) => topic.1.put(data).await,
-                None => {
-                    // TODO: this should be an error but the python FromSwarm is somewhat nondeterministic
-                    Ok(()) //Err("not subscribed to topic!".into()),
-                }
-            };
-            _ = result_sender.send(res);
+            let result = swarm
+                .behaviour_mut()
+                .gossipsub
+                .subscribe(&gossipsub::IdentTopic::new(topic));
+            _ = result_sender.send(result);
         }
         ToSwarm::Unsubscribe {
             topic,
             result_sender,
         } => {
-            let Some((_, (subscriber, publisher))) = topics.remove_entry(&topic) else {
-                _ = result_sender.send(false);
-                return;
-            };
-            _ = publisher.undeclare().await;
-            _ = subscriber.undeclare().await;
-            _ = result_sender.send(true);
+            let result = swarm
+                .behaviour_mut()
+                .gossipsub
+                .unsubscribe(&gossipsub::IdentTopic::new(topic));
+            _ = result_sender.send(result);
         }
-        ToSwarm::Subscribe {
+        ToSwarm::Publish {
             topic,
+            data,
             result_sender,
         } => {
-            assert!(topic.is_ascii());
-            if topics.contains_key(&topic) {
-                _ = result_sender.send(Ok(false));
-                return;
-            }
-
-            let publisher_res = session
-                .declare_publisher(format!("topics/{topic}"))
-                .congestion_control(CongestionControl::Block)
-                .await;
-            let publisher = match publisher_res {
-                Ok(p) => p,
-                Err(e) => {
-                    _ = result_sender.send(Err(e));
-                    return;
-                }
-            };
-
-            let subscriber_res = session
-                .declare_subscriber(format!("topics/{topic}"))
-                .allowed_origin(zenoh::sample::Locality::Remote)
-                .callback({
-                    let sender = to_topics.clone();
-                    let topic = topic.clone();
-                    move |sample| {
-                        if sample.kind() != SampleKind::Put {
-                            return;
-                        }
-                        _ = sender.try_send(FromSwarm::Message {
-                            topic: topic.clone(),
-                            data: sample.payload().to_bytes().to_vec(),
-                        });
-                    }
-                })
-                .await;
-            let subscriber = match subscriber_res {
-                Ok(s) => s,
-                Err(e) => {
-                    _ = result_sender.send(Err(e));
-                    return;
-                }
-            };
-
-            assert!(topics.insert(topic, (subscriber, publisher)).is_none());
-            _ = result_sender.send(Ok(true));
+            let result = swarm
+                .behaviour_mut()
+                .gossipsub
+                .publish(gossipsub::IdentTopic::new(topic), data);
+            _ = result_sender.send(result);
         }
     }
 }
 
-pub async fn create_swarm(
-    identity: &str,
-    namespace: &str,
-    from_client: mpsc::Receiver<ToSwarm>,
-    listen_port: u16,
-    discovery_service_port: u16,
-) -> Result<Swarm> {
-    let cfg = crate::cfg(identity, listen_port)?;
-    let session = crate::open(cfg, namespace, listen_port, discovery_service_port).await?;
-    Ok(Swarm {
-        session,
-        from_client,
-    })
+fn filter_swarm_event(event: SwarmEvent<BehaviourEvent>) -> Option<FromSwarm> {
+    match event {
+        SwarmEvent::Behaviour(BehaviourEvent::Gossipsub(gossipsub::Event::Message {
+            message:
+                gossipsub::Message {
+                    source: Some(peer_id),
+                    topic,
+                    data,
+                    ..
+                },
+            ..
+        })) => Some(FromSwarm::Message {
+            from: peer_id,
+            topic: topic.into_string(),
+            data,
+        }),
+        SwarmEvent::Behaviour(BehaviourEvent::Discovery(
+            discovery::Event::ConnectionEstablished { peer_id, .. },
+        )) => Some(FromSwarm::Discovered { peer_id }),
+        SwarmEvent::Behaviour(BehaviourEvent::Discovery(discovery::Event::ConnectionClosed {
+            peer_id,
+            ..
+        })) => Some(FromSwarm::Expired { peer_id }),
+        _ => None,
+    }
+}
+
+/// Create and configure a swarm.
+///
+/// - `listen_port`: TCP port to listen on. `0` lets the OS assign one.
+/// - `bootstrap_peers`: multiaddrs to dial for environments without mDNS.
+pub fn create_swarm(
+    keypair: identity::Keypair,
+    from_client: mpsc::Receiver<ToSwarm>,
+    bootstrap_peers: Vec<String>,
+    listen_port: u16,
+) -> alias::AnyResult<Swarm> {
+    let parsed_bootstrap_peers: Vec<libp2p::Multiaddr> = bootstrap_peers
+        .iter()
+        .filter(|s| !s.is_empty())
+        .filter_map(|s| s.parse().ok())
+        .collect();
+
+    let mut swarm = SwarmBuilder::with_existing_identity(keypair)
+        .with_tokio()
+        .with_other_transport(tcp_transport)?
+        .with_behaviour(|keypair| Behaviour::new(keypair, parsed_bootstrap_peers))?
+        .build();
+
+    swarm.listen_on(format!("/ip4/0.0.0.0/tcp/{listen_port}").parse()?)?;
+    Ok(Swarm { swarm, from_client })
+}
+
+mod transport {
+    use crate::alias;
+    use crate::swarm::{NETWORK_VERSION, OVERRIDE_VERSION_ENV_VAR};
+    use futures_lite::{AsyncRead, AsyncWrite};
+    use keccak_const::Sha3_256;
+    use libp2p::core::muxing;
+    use libp2p::core::transport::Boxed;
+    use libp2p::pnet::{PnetError, PnetOutput};
+    use libp2p::{PeerId, Transport, identity, noise, pnet, yamux};
+    use std::{env, sync::LazyLock};
+
+    /// Key used for networking's private network; parametrized on the [`NETWORK_VERSION`].
+    /// See [`pnet_upgrade`] for more.
+    static PNET_PRESHARED_KEY: LazyLock<[u8; 32]> = LazyLock::new(|| {
+        let builder = Sha3_256::new().update(b"exo_discovery_network");
+
+        if let Ok(var) = env::var(OVERRIDE_VERSION_ENV_VAR) {
+            let bytes = var.into_bytes();
+            builder.update(&bytes)
+        } else {
+            builder.update(NETWORK_VERSION)
+        }
+        .finalize()
+    });
+
+    /// Make the Swarm run on a private network, as to not clash with public libp2p nodes and
+    /// also different-versioned instances of this same network.
+    /// This is implemented as an additional "upgrade" ontop of existing [`libp2p::Transport`] layers.
+    async fn pnet_upgrade<TSocket>(
+        socket: TSocket,
+        _: impl Sized,
+    ) -> Result<PnetOutput<TSocket>, PnetError>
+    where
+        TSocket: AsyncRead + AsyncWrite + Send + Unpin + 'static,
+    {
+        use pnet::{PnetConfig, PreSharedKey};
+        PnetConfig::new(PreSharedKey::new(*PNET_PRESHARED_KEY))
+            .handshake(socket)
+            .await
+    }
+
+    /// TCP/IP transport layer configuration.
+    pub fn tcp_transport(
+        keypair: &identity::Keypair,
+    ) -> alias::AnyResult<Boxed<(PeerId, muxing::StreamMuxerBox)>> {
+        use libp2p::{
+            core::upgrade::Version,
+            tcp::{Config, tokio},
+        };
+
+        // `TCP_NODELAY` enabled => avoid latency
+        let tcp_config = Config::default().nodelay(true);
+
+        // V1 + lazy flushing => 0-RTT negotiation
+        let upgrade_version = Version::V1Lazy;
+
+        // Noise is faster than TLS + we don't care much for security
+        let noise_config = noise::Config::new(keypair)?;
+
+        // Use default Yamux config for multiplexing
+        let yamux_config = yamux::Config::default();
+
+        // Create new Tokio-driven TCP/IP transport layer
+        let base_transport = tokio::Transport::new(tcp_config)
+            .and_then(pnet_upgrade)
+            .upgrade(upgrade_version)
+            .authenticate(noise_config)
+            .multiplex(yamux_config);
+
+        // Return boxed transport (to flatten complex type)
+        Ok(base_transport.boxed())
+    }
+}
+
+mod behaviour {
+    use crate::{alias, discovery};
+    use libp2p::swarm::NetworkBehaviour;
+    use libp2p::{gossipsub, identity};
+
+    /// Behavior of the Swarm which composes all desired behaviors:
+    /// Right now its just [`discovery::Behaviour`] and [`gossipsub::Behaviour`].
+    #[derive(NetworkBehaviour)]
+    pub struct Behaviour {
+        pub discovery: discovery::Behaviour,
+        pub gossipsub: gossipsub::Behaviour,
+    }
+
+    impl Behaviour {
+        pub fn new(
+            keypair: &identity::Keypair,
+            bootstrap_peers: Vec<libp2p::Multiaddr>,
+        ) -> alias::AnyResult<Self> {
+            Ok(Self {
+                discovery: discovery::Behaviour::new(keypair, bootstrap_peers)?,
+                gossipsub: gossipsub_behaviour(keypair),
+            })
+        }
+    }
+
+    fn gossipsub_behaviour(keypair: &identity::Keypair) -> gossipsub::Behaviour {
+        use gossipsub::{ConfigBuilder, MessageAuthenticity, ValidationMode};
+
+        // build a gossipsub network behaviour
+        //  => signed message authenticity + strict validation mode means the message-ID is
+        //     automatically provided by gossipsub w/out needing to provide custom message-ID function
+        gossipsub::Behaviour::new(
+            MessageAuthenticity::Signed(keypair.clone()),
+            ConfigBuilder::default()
+                .max_transmit_size(8 * 1024 * 1024)
+                .validation_mode(ValidationMode::Strict)
+                .build()
+                .expect("the configuration should always be valid"),
+        )
+        .expect("creating gossipsub behavior should always work")
+    }
 }

rust/networking/tests/bootstrap_peers.rs

@@ -0,0 +1,107 @@
+use futures_lite::StreamExt;
+use networking::swarm::{FromSwarm, create_swarm};
+use std::time::Duration;
+use tokio::sync::mpsc;
+use tokio::time::timeout;
+
+/// Helper: find a free TCP port.
+fn free_port() -> u16 {
+    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+    listener.local_addr().unwrap().port()
+}
+
+/// Two nodes connect via bootstrap peers — no mDNS needed.
+///
+/// Node A listens on a fixed port. Node B bootstraps to A's address.
+/// We verify that B emits `FromSwarm::Discovered` for A's peer ID.
+#[tokio::test]
+async fn two_nodes_connect_via_bootstrap_peers() {
+    let port_a = free_port();
+
+    // Node A: listens on a known port, no bootstrap peers
+    let keypair_a = libp2p::identity::Keypair::generate_ed25519();
+    let peer_id_a = keypair_a.public().to_peer_id();
+    let (_tx_a, rx_a) = mpsc::channel(16);
+    let swarm_a = create_swarm(keypair_a, rx_a, vec![], port_a).expect("create swarm A");
+    let mut stream_a = swarm_a.into_stream();
+
+    // Node B: bootstraps to A's address
+    let keypair_b = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx_b, rx_b) = mpsc::channel(16);
+    let swarm_b = create_swarm(
+        keypair_b,
+        rx_b,
+        vec![format!("/ip4/127.0.0.1/tcp/{port_a}")],
+        0,
+    )
+    .expect("create swarm B");
+    let mut stream_b = swarm_b.into_stream();
+
+    // Wait for B to discover A (connection established)
+    let connected = timeout(Duration::from_secs(10), async {
+        loop {
+            tokio::select! {
+                Some(event) = stream_a.next() => {
+                    // A will also see B connect, but we check from B's perspective
+                    let _ = event;
+                }
+                Some(event) = stream_b.next() => {
+                    if let FromSwarm::Discovered { peer_id } = event {
+                        if peer_id == peer_id_a {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+    })
+    .await;
+
+    assert!(
+        connected.is_ok() && connected.unwrap(),
+        "Node B should discover Node A via bootstrap peer"
+    );
+}
+
+/// Empty bootstrap peers should work (backward compatible).
+#[tokio::test]
+async fn create_swarm_with_empty_bootstrap_peers() {
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(keypair, rx, vec![], 0);
+    assert!(
+        swarm.is_ok(),
+        "create_swarm with no bootstrap peers should succeed"
+    );
+}
+
+/// Invalid multiaddr strings are silently filtered out.
+#[tokio::test]
+async fn create_swarm_ignores_invalid_bootstrap_addrs() {
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(
+        keypair,
+        rx,
+        vec![
+            "not-a-valid-multiaddr".to_string(),
+            "".to_string(),
+            "/ip4/10.0.0.1/tcp/30000".to_string(), // valid
+        ],
+        0,
+    );
+    assert!(
+        swarm.is_ok(),
+        "create_swarm should succeed even with invalid bootstrap addrs"
+    );
+}
+
+/// Fixed listen port works correctly.
+#[tokio::test]
+async fn create_swarm_with_fixed_port() {
+    let port = free_port();
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(keypair, rx, vec![], port);
+    assert!(swarm.is_ok(), "create_swarm with fixed port should succeed");
+}

rust/networking/tests/dummy.rs

@@ -0,0 +1,7 @@
+// maybe this will hold test in the future...??
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn does_nothing() {}
+}

rust/parts.nix

@@ -1,7 +1,7 @@
 { inputs, ... }:
 {
   perSystem =
-    { inputs', pkgs, lib, ... }:
+    { inputs', self', pkgs, lib, ... }:
     let
       # Fenix nightly toolchain with all components
       rustToolchain = inputs'.fenix.packages.stable.withComponents [
@@ -55,7 +55,6 @@
         ];
 
         OPENSSL_NO_VENDOR = "1";
-        MATURIN_NO_INSTALL_RUST = "1";
 
         # Required for pyo3 tests to find libpython
         LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.python313 ];
@@ -80,13 +79,13 @@
       };
 
       config = {
-        packages = {
+        packages = rec {
           # Python bindings wheel via maturin
-          exo-rs = craneLib.buildPackage (
+          exo_pyo3_bindings = craneLib.buildPackage (
             commonArgs
             // {
               inherit cargoArtifacts;
-              pname = "exo-rs";
+              pname = "exo_pyo3_bindings";
 
               nativeBuildInputs = commonArgs.nativeBuildInputs ++ [
                 pkgs.maturin
@@ -96,7 +95,7 @@
                 maturin build \
                   --release \
                   --manylinux off \
-                  --manifest-path rust/exo_rs/Cargo.toml \
+                  --manifest-path rust/exo_pyo3_bindings/Cargo.toml \
                   --features "pyo3/extension-module,pyo3/experimental-async" \
                   --interpreter ${pkgs.python313}/bin/python \
                   --out dist
@@ -111,6 +110,19 @@
               '';
             }
           );
+          babblerd-unwrapped = craneLib.buildPackage (
+            commonArgs // {
+              inherit cargoArtifacts;
+              pname = "babblerd-unwrapped";
+            }
+          );
+          babblerd = pkgs.writeShellApplication {
+            name = "babblerd";
+            runtimeInputs = [ self'.packages.babeld ];
+            text = ''
+              exec ${babblerd-unwrapped}/bin/babblerd "$@"
+            '';
+          };
         };
 
         checks = {

rust/util/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "util"
+version = { workspace = true }
+edition = { workspace = true }
+publish = false
+
+[lib]
+doctest = false
+name = "util"
+path = "src/lib.rs"
+
+[lints]
+workspace = true
+
+[dependencies]

rust/util/src/lib.rs

@@ -0,0 +1 @@
+pub mod wakerdeque;

rust/util/src/wakerdeque.rs

@@ -0,0 +1,55 @@
+use std::collections::VecDeque;
+use std::fmt::{Debug, Formatter};
+use std::task::{Context, Waker};
+
+/// A wrapper around [`VecDeque`] which wakes (if it can) on any `push_*` methods,
+/// and updates the internally stored waker by consuming [`Context`] on any `pop_*` methods.
+pub struct WakerDeque<T> {
+    waker: Option<Waker>,
+    deque: VecDeque<T>,
+}
+
+impl<T: Debug> Debug for WakerDeque<T> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        self.deque.fmt(f)
+    }
+}
+
+impl<T> WakerDeque<T> {
+    pub fn new() -> Self {
+        Self {
+            waker: None,
+            deque: VecDeque::new(),
+        }
+    }
+
+    fn update(&mut self, cx: &mut Context<'_>) {
+        self.waker = Some(cx.waker().clone());
+    }
+
+    fn wake(&mut self) {
+        let Some(ref mut w) = self.waker else { return };
+        w.wake_by_ref();
+        self.waker = None;
+    }
+
+    pub fn pop_front(&mut self, cx: &mut Context<'_>) -> Option<T> {
+        self.update(cx);
+        self.deque.pop_front()
+    }
+
+    pub fn pop_back(&mut self, cx: &mut Context<'_>) -> Option<T> {
+        self.update(cx);
+        self.deque.pop_back()
+    }
+
+    pub fn push_front(&mut self, value: T) {
+        self.wake();
+        self.deque.push_front(value);
+    }
+
+    pub fn push_back(&mut self, value: T) {
+        self.wake();
+        self.deque.push_back(value);
+    }
+}

scripts/test_babbler/client.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import argparse
+import socket
+import sys
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(
+        description="IPv6 UDP client with optional explicit source bind"
+    )
+    p.add_argument("--dest", required=True, help="Destination IPv6 address")
+    p.add_argument("--port", type=int, default=45679, help="Destination UDP port")
+    p.add_argument("--source", help="Optional source IPv6 address to bind to")
+    p.add_argument("--message", default="hello", help="Payload to send")
+    p.add_argument(
+        "--timeout", type=float, default=5.0, help="Receive timeout in seconds"
+    )
+    args = p.parse_args()
+
+    s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    s.settimeout(args.timeout)
+
+    if args.source:
+        s.bind((args.source, 0, 0, 0))
+
+    print(f"local-before-send={s.getsockname()}")
+    s.sendto(args.message.encode(), (args.dest, args.port, 0, 0))
+    print(f"sent to=[{args.dest}]:{args.port}")
+    print(f"local-after-send={s.getsockname()}")
+
+    data, peer = s.recvfrom(65535)
+    print(f"from={peer} data={data!r}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_babbler/server.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import argparse
+import socket
+import sys
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(
+        description="IPv6 UDP server bound to a specific local address"
+    )
+    p.add_argument(
+        "--bind", required=True, help="Local IPv6 address to bind to, e.g. fde0:..."
+    )
+    p.add_argument("--port", type=int, default=45679, help="UDP port to listen on")
+    p.add_argument("--reply", default="ok", help="Reply prefix")
+    args = p.parse_args()
+
+    s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    s.bind((args.bind, args.port, 0, 0))
+
+    print(f"listening on [{args.bind}]:{args.port}")
+    print(f"sockname={s.getsockname()}")
+
+    data, peer = s.recvfrom(65535)
+    print(f"from={peer} data={data!r}")
+
+    out = args.reply.encode() + b":" + data
+    s.sendto(out, peer)
+    print(f"sent={out!r} to={peer}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/exo/__init__.py

@@ -1,3 +0,0 @@
-from importlib.metadata import version
-
-__version__ = version("exo")

src/exo/api/main.py

@@ -20,7 +20,7 @@ from fastapi.staticfiles import StaticFiles
 from hypercorn.asyncio import serve  # pyright: ignore[reportUnknownVariableType]
 from hypercorn.config import Config
 from hypercorn.typing import ASGIFramework
-from hypercorn.utils import LifespanTimeoutError, ShutdownError
+from hypercorn.utils import LifespanTimeoutError
 from loguru import logger
 
 from exo.api.adapters.chat_completions import (
@@ -50,8 +50,6 @@ from exo.api.keepalive import with_sse_keepalive
 from exo.api.types import (
     AddCustomModelParams,
     AdvancedImageParams,
-    AwaitInstanceReadyMessage,
-    AwaitInstanceTimeoutMessage,
     BenchChatCompletionRequest,
     BenchChatCompletionResponse,
     BenchImageGenerationResponse,
@@ -346,7 +344,6 @@ class API:
         self.app.post("/place_instance")(self.place_instance)
         self.app.get("/instance/placement")(self.get_placement)
         self.app.get("/instance/previews")(self.get_placement_previews)
-        self.app.get("/instance/await", response_model=None)(self.await_instance)
         self.app.get("/instance/{instance_id}")(self.get_instance)
         self.app.delete("/instance/{instance_id}")(self.delete_instance)
         self.app.get("/v1/instance-links")(self.list_instance_links)
@@ -636,48 +633,6 @@ class API:
             raise HTTPException(status_code=404, detail="Instance not found")
         return self.state.instances[instance_id]
 
-    async def await_instance(
-        self,
-        model_id: ModelId,
-        timeout_seconds: float = Query(default=0.0, ge=0.0, le=300.0),
-    ) -> StreamingResponse:
-        _sleep = 0.1
-
-        async def _stream() -> AsyncGenerator[str, None]:
-            deadline = (
-                None if timeout_seconds == 0 else anyio.current_time() + timeout_seconds
-            )
-
-            while True:
-                for instance in self.state.instances.values():
-                    if instance.shard_assignments.model_id == model_id:
-                        payload = AwaitInstanceReadyMessage(instance=instance)
-                        yield f"data: {payload.model_dump_json()}\n\n"
-                        return
-
-                if deadline is None:
-                    await anyio.sleep(_sleep)
-                else:
-                    remaining = deadline - anyio.current_time()
-                    if remaining <= 0:
-                        payload = AwaitInstanceTimeoutMessage(
-                            message=f"No instance found for model {model_id}"
-                        )
-                        yield f"data: {payload.model_dump_json()}\n\n"
-                        return
-
-                    await anyio.sleep(min(_sleep, remaining))
-
-        return StreamingResponse(
-            with_sse_keepalive(_stream()),
-            media_type="text/event-stream",
-            headers={
-                "Cache-Control": "no-cache",
-                "Connection": "close",
-                "X-Accel-Buffering": "no",
-            },
-        )
-
     async def delete_instance(self, instance_id: InstanceId) -> DeleteInstanceResponse:
         if instance_id not in self.state.instances:
             raise HTTPException(status_code=404, detail="Instance not found")
@@ -806,8 +761,6 @@ class API:
                 if isinstance(chunk, PrefillProgressChunk):
                     continue
 
-                sampler.mark_prefill_done()
-
                 if chunk.finish_reason == "error":
                     raise HTTPException(
                         status_code=500,
@@ -918,8 +871,10 @@ class API:
     ) -> ChatCompletionResponse | StreamingResponse:
         """OpenAI Chat Completions API - adapter."""
         task_params = await chat_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(task_params.model)
-        task_params = task_params.model_copy(update={"model": validated_model})
+        resolved_model = await self._resolve_and_validate_text_model(
+            ModelId(task_params.model)
+        )
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         command = await self._send_text_generation_with_images(task_params)
 
@@ -951,10 +906,10 @@ class API:
         self, payload: BenchChatCompletionRequest
     ) -> BenchChatCompletionResponse | StreamingResponse:
         task_params = await chat_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(
+        resolved_model = await self._resolve_and_validate_text_model(
             ModelId(task_params.model)
         )
-        task_params = task_params.model_copy(update={"model": validated_model})
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         task_params = task_params.model_copy(
             update={
@@ -984,10 +939,8 @@ class API:
 
         return await self._collect_text_generation_with_stats(command.command_id)
 
-    async def _validate_model_has_instance(self, model_id: ModelId) -> ModelId:
-        """Validate a model has an active instance.
-        If the model isn't even downloaded, triggers notification to user to download model.
-
+    async def _resolve_and_validate_text_model(self, model_id: ModelId) -> ModelId:
+        """Validate a text model exists and return the resolved model ID.
 
         Raises HTTPException 404 if no instance is found for the model.
         """
@@ -995,21 +948,30 @@ class API:
             instance.shard_assignments.model_id == model_id
             for instance in self.state.instances.values()
         ):
-            # Check if model is actually downloaded
-            model_is_downloaded = any(
-                isinstance(download, DownloadCompleted)
-                and download.shard_metadata.model_card.model_id == model_id
-                for node_downloads in self.state.downloads.values()
-                for download in node_downloads
-            )
-            if not model_is_downloaded:
-                await self._trigger_notify_user_to_download_model(model_id)
-
+            await self._trigger_notify_user_to_download_model(model_id)
             raise HTTPException(
-                status_code=404, detail=f"No instance found for model {model_id}"
+                status_code=404,
+                detail=f"No instance found for model {model_id}",
             )
         return model_id
 
+    async def _validate_image_model(self, model: ModelId) -> ModelId:
+        """Validate model exists and return resolved model ID.
+
+        Raises HTTPException 404 if no instance is found for the model.
+        """
+        model_card = await ModelCard.load(model)
+        resolved_model = model_card.model_id
+        if not any(
+            instance.shard_assignments.model_id == resolved_model
+            for instance in self.state.instances.values()
+        ):
+            await self._trigger_notify_user_to_download_model(resolved_model)
+            raise HTTPException(
+                status_code=404, detail=f"No instance found for model {resolved_model}"
+            )
+        return resolved_model
+
     def stream_events(self) -> StreamingResponse:
         def _generate_json_array(events: Iterable[Event]) -> Iterable[str]:
             yield "["
@@ -1062,9 +1024,7 @@ class API:
         """
         payload = payload.model_copy(
             update={
-                "model": await self._validate_model_has_instance(
-                    ModelId(payload.model)
-                ),
+                "model": await self._validate_image_model(ModelId(payload.model)),
                 "advanced_params": _ensure_seed(payload.advanced_params),
             }
         )
@@ -1332,9 +1292,7 @@ class API:
     ) -> BenchImageGenerationResponse:
         payload = payload.model_copy(
             update={
-                "model": await self._validate_model_has_instance(
-                    ModelId(payload.model)
-                ),
+                "model": await self._validate_image_model(ModelId(payload.model)),
                 "stream": False,
                 "partial_images": 0,
                 "advanced_params": _ensure_seed(payload.advanced_params),
@@ -1370,7 +1328,7 @@ class API:
         advanced_params: AdvancedImageParams | None,
     ) -> ImageEdits:
         """Prepare and send an image edits command with chunked image upload."""
-        validated_model = await self._validate_model_has_instance(model)
+        resolved_model = await self._validate_image_model(model)
         advanced_params = _ensure_seed(advanced_params)
 
         image_content = await image.read()
@@ -1389,7 +1347,7 @@ class API:
                 image_data="",
                 total_input_chunks=total_chunks,
                 prompt=prompt,
-                model=validated_model,
+                model=resolved_model,
                 n=n,
                 size=size,
                 response_format=response_format,
@@ -1410,7 +1368,7 @@ class API:
             await self._send(
                 SendInputChunk(
                     chunk=InputImageChunk(
-                        model=validated_model,
+                        model=resolved_model,
                         command_id=command.command_id,
                         data=chunk_data,
                         chunk_index=chunk_index,
@@ -1534,10 +1492,10 @@ class API:
     ) -> ClaudeMessagesResponse | StreamingResponse:
         """Claude Messages API - adapter."""
         task_params = await claude_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(
+        resolved_model = await self._resolve_and_validate_text_model(
             ModelId(task_params.model)
         )
-        task_params = task_params.model_copy(update={"model": validated_model})
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         command = await self._send_text_generation_with_images(task_params)
 
@@ -1572,8 +1530,8 @@ class API:
     ) -> ResponsesResponse | StreamingResponse:
         """OpenAI Responses API."""
         task_params = await responses_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(task_params.model)
-        task_params = task_params.model_copy(update={"model": validated_model})
+        resolved_model = await self._resolve_and_validate_text_model(task_params.model)
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         command = await self._send_text_generation_with_images(task_params)
 
@@ -1615,10 +1573,10 @@ class API:
         body = await request.body()
         payload = OllamaChatRequest.model_validate_json(body)
         task_params = ollama_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(
+        resolved_model = await self._resolve_and_validate_text_model(
             ModelId(task_params.model)
         )
-        task_params = task_params.model_copy(update={"model": validated_model})
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         command = await self._send_text_generation_with_images(task_params)
 
@@ -1651,10 +1609,10 @@ class API:
         body = await request.body()
         payload = OllamaGenerateRequest.model_validate_json(body)
         task_params = ollama_generate_request_to_text_generation(payload)
-        validated_model = await self._validate_model_has_instance(
+        resolved_model = await self._resolve_and_validate_text_model(
             ModelId(task_params.model)
         )
-        task_params = task_params.model_copy(update={"model": validated_model})
+        task_params = task_params.model_copy(update={"model": resolved_model})
 
         command = await self._send_text_generation_with_images(task_params)
 
@@ -1956,10 +1914,6 @@ class API:
                     cfg,
                     shutdown_trigger=ev.wait,
                 )
-                if not ev.is_set():
-                    raise ShutdownError(
-                        "Server exited without shutdown trigger - exiting abnormally"
-                    )
             except LifespanTimeoutError as e:
                 logger.warning(
                     "Graceful server shutdown timed out, some connections forcebly closed"

src/exo/api/types/__init__.py

@@ -1,7 +1,5 @@
 from .api import AddCustomModelParams as AddCustomModelParams
 from .api import AdvancedImageParams as AdvancedImageParams
-from .api import AwaitInstanceReadyMessage as AwaitInstanceReadyMessage
-from .api import AwaitInstanceTimeoutMessage as AwaitInstanceTimeoutMessage
 from .api import BenchChatCompletionRequest as BenchChatCompletionRequest
 from .api import BenchChatCompletionResponse as BenchChatCompletionResponse
 from .api import BenchImageGenerationResponse as BenchImageGenerationResponse

src/exo/api/types/api.py

@@ -186,12 +186,6 @@ class NodePowerStats(BaseModel, frozen=True):
     node_id: NodeId
     samples: int
     avg_sys_power: float
-    # Per-phase breakdown. Populated only when the caller marks a phase
-    # boundary (e.g. prefill -> generation); None otherwise.
-    prefill_avg_sys_power: float | None = None
-    generation_avg_sys_power: float | None = None
-    prefill_energy_joules: float | None = None
-    generation_energy_joules: float | None = None
 
 
 class PowerUsage(BaseModel, frozen=True):
@@ -199,16 +193,6 @@ class PowerUsage(BaseModel, frozen=True):
     nodes: list[NodePowerStats]
     total_avg_sys_power_watts: float
     total_energy_joules: float
-    # Split between the prefill (prompt-processing) phase and the
-    # generation/decode phase. Populated only when the caller marks a phase
-    # boundary; None otherwise. The two phase energies should sum to
-    # approximately `total_energy_joules` (modulo interpolation rounding).
-    prefill_seconds: float | None = None
-    generation_seconds: float | None = None
-    prefill_energy_joules: float | None = None
-    generation_energy_joules: float | None = None
-    prefill_avg_sys_power_watts: float | None = None
-    generation_avg_sys_power_watts: float | None = None
 
 
 class BenchChatCompletionResponse(ChatCompletionResponse):
@@ -307,16 +291,6 @@ class DeleteInstanceResponse(BaseModel):
     instance_id: InstanceId
 
 
-class AwaitInstanceReadyMessage(BaseModel):
-    type: Literal["ready"] = "ready"
-    instance: Instance
-
-
-class AwaitInstanceTimeoutMessage(BaseModel):
-    type: Literal["timeout"] = "timeout"
-    message: str
-
-
 class CancelCommandResponse(BaseModel):
     message: str
     command_id: CommandId

src/exo/download/coordinator.py

@@ -15,10 +15,6 @@ from exo.download.download_utils import (
     resolve_existing_model,
 )
 from exo.download.shard_downloader import ShardDownloader
-from exo.routing.event_router import (
-    EventRouterBrokenResourceError,
-    EventRouterClosedResourceError,
-)
 from exo.shared.constants import EXO_DEFAULT_MODELS_DIR, EXO_MODELS_READ_ONLY_DIRS
 from exo.shared.models import model_cards
 from exo.shared.models.model_cards import ModelId
@@ -143,14 +139,7 @@ class DownloadCoordinator:
             async with self._tg as tg:
                 tg.start_soon(self._command_processor)
                 tg.start_soon(self._emit_existing_download_progress)
-        except* (EventRouterBrokenResourceError, EventRouterClosedResourceError):
-            # Event router has been closed (try-star syntax handles error groups)
-            pass
         finally:
-            # don't forget to clean up resources
-            self.download_command_receiver.close()
-            self.event_sender.close()
-
             self._stopped.set()
 
     async def shutdown(self) -> None: