migrate model cards to .toml files

.github/workflows/build-app.yml

@@ -1,16 +1,5 @@
 name: Build EXO macOS DMG
 
-# Release workflow:
-# 1. Create a draft GitHub Release with the tag name (e.g. v1.0.0) and write release notes in markdown
-# 2. Push the tag: git tag v1.0.0 && git push origin v1.0.0
-# 3. This workflow builds, signs, and notarizes the DMG
-# 4. Release notes are embedded in appcast.xml for Sparkle (rendered as markdown)
-# 5. DMG and appcast.xml are uploaded to S3
-# 6. The draft GitHub Release is published with the DMG attached
-#
-# For alpha releases (e.g. v1.0.0-alpha.1): draft release and notes are optional.
-# If no draft exists, a release is auto-created with generated notes.
-
 on:
   workflow_dispatch:
   push:
@@ -22,10 +11,8 @@ on:
 jobs:
   build-macos-app:
     runs-on: "macos-26"
-    permissions:
-      contents: write
     env:
-      SPARKLE_VERSION: 2.9.0-beta.1
+      SPARKLE_VERSION: 2.8.1
       SPARKLE_DOWNLOAD_PREFIX: ${{ secrets.SPARKLE_DOWNLOAD_PREFIX }}
       SPARKLE_FEED_URL: ${{ secrets.SPARKLE_FEED_URL }}
       SPARKLE_ED25519_PUBLIC: ${{ secrets.SPARKLE_ED25519_PUBLIC }}
@@ -100,52 +87,6 @@ jobs:
             exit 1
           fi
 
-      - name: Fetch and validate release notes
-        if: github.ref_type == 'tag'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Find draft release by name using gh release list (more reliable with default token)
-          echo "Looking for draft release named '$GITHUB_REF_NAME'..."
-          DRAFT_EXISTS=$(gh release list --json name,isDraft --jq ".[] | select(.isDraft == true) | select(.name == \"$GITHUB_REF_NAME\") | .name" 2>/dev/null || echo "")
-
-          if [[ -z "$DRAFT_EXISTS" ]]; then
-            if [[ "$IS_ALPHA" == "true" ]]; then
-              echo "No draft release found for alpha tag $GITHUB_REF_NAME (optional for alphas)"
-              echo "HAS_RELEASE_NOTES=false" >> $GITHUB_ENV
-              exit 0
-            fi
-            echo "ERROR: No draft release found for tag $GITHUB_REF_NAME"
-            echo "Please create a draft release with release notes before pushing the tag."
-            exit 1
-          fi
-
-          # Fetch full release details via API to get body and ID
-          echo "Found draft release, fetching details..."
-          RELEASE_JSON=$(gh api repos/${{ github.repository }}/releases --jq ".[] | select(.draft == true) | select(.name == \"$GITHUB_REF_NAME\")" 2>/dev/null || echo "")
-
-          # Extract release notes
-          NOTES=$(echo "$RELEASE_JSON" | jq -r '.body // ""')
-          if [[ -z "$NOTES" || "$NOTES" == "null" ]]; then
-            if [[ "$IS_ALPHA" == "true" ]]; then
-              echo "Draft release has no notes (optional for alphas)"
-              echo "HAS_RELEASE_NOTES=false" >> $GITHUB_ENV
-              exit 0
-            fi
-            echo "ERROR: Draft release exists but has no release notes"
-            echo "Please add release notes to the draft release before pushing the tag."
-            exit 1
-          fi
-
-          # Save release ID for later publishing
-          RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
-          echo "DRAFT_RELEASE_ID=$RELEASE_ID" >> $GITHUB_ENV
-          echo "HAS_RELEASE_NOTES=true" >> $GITHUB_ENV
-
-          echo "Found draft release (ID: $RELEASE_ID), saving release notes..."
-          echo "$NOTES" > /tmp/release_notes.md
-          echo "RELEASE_NOTES_FILE=/tmp/release_notes.md" >> $GITHUB_ENV
-
       # ============================================================
       # Install dependencies
       # ============================================================
@@ -363,28 +304,6 @@ jobs:
             $CHANNEL_FLAG \
             .
 
-      - name: Inject release notes into appcast
-        if: github.ref_type == 'tag' && env.HAS_RELEASE_NOTES == 'true'
-        env:
-          RELEASE_VERSION: ${{ env.RELEASE_VERSION }}
-        run: |
-          # Inject markdown release notes with sparkle:format="markdown" (Sparkle 2.9+)
-          export NOTES=$(cat "$RELEASE_NOTES_FILE")
-
-          # Insert description after the enclosure tag for this version
-          awk '
-            /<enclosure[^>]*>/ && index($0, ENVIRON["RELEASE_VERSION"]) {
-              print
-              print "            <description sparkle:format=\"markdown\"><![CDATA["
-              print ENVIRON["NOTES"]
-              print "            ]]></description>"
-              next
-            }
-            { print }
-          ' output/appcast.xml > output/appcast.xml.tmp && mv output/appcast.xml.tmp output/appcast.xml
-
-          echo "Injected markdown release notes for version $RELEASE_VERSION"
-
       # ============================================================
       # Upload artifacts
       # ============================================================
@@ -417,26 +336,3 @@ jobs:
             aws s3 cp "$DMG_NAME" "s3://${SPARKLE_S3_BUCKET}/${PREFIX}EXO-latest.dmg"
             aws s3 cp appcast.xml "s3://${SPARKLE_S3_BUCKET}/${PREFIX}appcast.xml" --content-type application/xml --cache-control no-cache
           fi
-
-      - name: Publish GitHub Release
-        if: github.ref_type == 'tag'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          DMG_PATH="output/EXO-${RELEASE_VERSION}.dmg"
-
-          if [[ "$HAS_RELEASE_NOTES" == "true" ]]; then
-            # Update the draft release with the tag and upload DMG
-            gh api --method PATCH "repos/${{ github.repository }}/releases/$DRAFT_RELEASE_ID" \
-              -f tag_name="$GITHUB_REF_NAME" \
-              -F draft=false
-            gh release upload "$GITHUB_REF_NAME" "$DMG_PATH" --clobber
-            echo "Published release $GITHUB_REF_NAME with DMG attached"
-          else
-            # Alpha without draft release - create one with auto-generated notes
-            gh release create "$GITHUB_REF_NAME" "$DMG_PATH" \
-              --title "$GITHUB_REF_NAME" \
-              --generate-notes \
-              --prerelease
-            echo "Created alpha release $GITHUB_REF_NAME with auto-generated notes"
-          fi

AGENTS.md

@@ -40,31 +40,6 @@ uv run ruff check
 nix fmt
 ```
 
-## Pre-Commit Checks (REQUIRED)
-
-**IMPORTANT: Always run these checks before committing code. CI will fail if these don't pass.**
-
-```bash
-# 1. Type checking - MUST pass with 0 errors
-uv run basedpyright
-
-# 2. Linting - MUST pass
-uv run ruff check
-
-# 3. Formatting - MUST be applied
-nix fmt
-
-# 4. Tests - MUST pass
-uv run pytest
-```
-
-Run all checks in sequence:
-```bash
-uv run basedpyright && uv run ruff check && nix fmt && uv run pytest
-```
-
-If `nix fmt` changes any files, stage them before committing. The CI runs `nix flake check` which verifies formatting, linting, and runs Rust tests.
-
 ## Architecture
 
 ### Node Composition

app/EXO/EXO.xcodeproj/project.pbxproj

@@ -585,7 +585,7 @@
 			repositoryURL = "https://github.com/sparkle-project/Sparkle.git";
 			requirement = {
 				kind = upToNextMajorVersion;
-				minimumVersion = 2.9.0-beta.1;
+				minimumVersion = 2.8.1;
 			};
 		};
 /* End XCRemoteSwiftPackageReference section */

app/EXO/EXO.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -6,8 +6,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/sparkle-project/Sparkle.git",
       "state" : {
-        "revision" : "e641adb41915a8409895e2e30666aa64e487b637",
-        "version" : "2.9.0-beta.1"
+        "revision" : "5581748cef2bae787496fe6d61139aebe0a451f6",
+        "version" : "2.8.1"
       }
     }
   ],

app/EXO/EXO/ContentView.swift

@@ -56,11 +56,6 @@ struct ContentView: View {
     }
 
     private var shouldShowLocalNetworkWarning: Bool {
-        // Show warning if local network is not working and EXO is running.
-        // The checker uses a longer timeout on first launch to allow time for
-        // the permission prompt, so this correctly handles both:
-        // 1. User denied permission on first launch
-        // 2. Permission broke after restart (macOS TCC bug)
         if case .notWorking = localNetworkChecker.status {
             return controller.status != .stopped
         }

app/EXO/EXO/Services/LocalNetworkChecker.swift

@@ -5,8 +5,8 @@ import os.log
 /// Checks if the app's local network permission is actually functional.
 ///
 /// macOS local network permission can appear enabled in System Preferences but not
-/// actually work after a restart. This service uses NWConnection to mDNS multicast
-/// to verify actual connectivity.
+/// actually work after a restart. This service detects this by creating a UDP
+/// connection to the mDNS multicast address (224.0.0.251:5353).
 @MainActor
 final class LocalNetworkChecker: ObservableObject {
     enum Status: Equatable {
@@ -35,43 +35,30 @@ final class LocalNetworkChecker: ObservableObject {
     }
 
     private static let logger = Logger(subsystem: "io.exo.EXO", category: "LocalNetworkChecker")
-    private static let hasCompletedInitialCheckKey = "LocalNetworkChecker.hasCompletedInitialCheck"
 
     @Published private(set) var status: Status = .unknown
+    @Published private(set) var lastConnectionState: String = "none"
 
     private var connection: NWConnection?
     private var checkTask: Task<Void, Never>?
 
-    /// Whether we've completed at least one check (stored in UserDefaults)
-    private var hasCompletedInitialCheck: Bool {
-        get { UserDefaults.standard.bool(forKey: Self.hasCompletedInitialCheckKey) }
-        set { UserDefaults.standard.set(newValue, forKey: Self.hasCompletedInitialCheckKey) }
-    }
-
     /// Checks if local network access is working.
     func check() {
         checkTask?.cancel()
         status = .checking
-
-        // Use longer timeout on first launch to allow time for permission prompt
-        let isFirstCheck = !hasCompletedInitialCheck
-        let timeout: UInt64 = isFirstCheck ? 30_000_000_000 : 3_000_000_000
+        lastConnectionState = "connecting"
 
         checkTask = Task { [weak self] in
             guard let self else { return }
-
-            Self.logger.info("Checking local network connectivity (first check: \(isFirstCheck))")
-            let result = await self.checkConnectivity(timeout: timeout)
+            let result = await self.performCheck()
             self.status = result
-            self.hasCompletedInitialCheck = true
-
             Self.logger.info("Local network check complete: \(result.displayText)")
         }
     }
 
-    /// Checks connectivity using NWConnection to mDNS multicast.
-    /// The connection attempt triggers the permission prompt if not yet shown.
-    private func checkConnectivity(timeout: UInt64) async -> Status {
+    private func performCheck() async -> Status {
+        Self.logger.info("Checking local network access via UDP multicast")
+
         connection?.cancel()
         connection = nil
 
@@ -97,7 +84,22 @@ final class LocalNetworkChecker: ObservableObject {
                 continuation.resume(returning: status)
             }
 
-            conn.stateUpdateHandler = { state in
+            conn.stateUpdateHandler = { [weak self] state in
+                let stateStr: String
+                switch state {
+                case .setup: stateStr = "setup"
+                case .preparing: stateStr = "preparing"
+                case .ready: stateStr = "ready"
+                case .waiting(let e): stateStr = "waiting(\(e))"
+                case .failed(let e): stateStr = "failed(\(e))"
+                case .cancelled: stateStr = "cancelled"
+                @unknown default: stateStr = "unknown"
+                }
+
+                Task { @MainActor in
+                    self?.lastConnectionState = stateStr
+                }
+
                 switch state {
                 case .ready:
                     resumeOnce(.working)
@@ -106,7 +108,6 @@ final class LocalNetworkChecker: ObservableObject {
                     if errorStr.contains("54") || errorStr.contains("ECONNRESET") {
                         resumeOnce(.notWorking(reason: "Connection blocked"))
                     }
-                // Otherwise keep waiting - might be showing permission prompt
                 case .failed(let error):
                     let errorStr = "\(error)"
                     if errorStr.contains("65") || errorStr.contains("EHOSTUNREACH")
@@ -126,7 +127,7 @@ final class LocalNetworkChecker: ObservableObject {
             conn.start(queue: .main)
 
             Task {
-                try? await Task.sleep(nanoseconds: timeout)
+                try? await Task.sleep(nanoseconds: 3_000_000_000)
                 let state = conn.state
                 switch state {
                 case .ready:

bench/exo_bench.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import argparse
-import contextlib
 import http.client
 import json
 import os
@@ -27,7 +26,7 @@ class ExoHttpError(RuntimeError):
 
 
 class ExoClient:
-    def __init__(self, host: str, port: int, timeout_s: float = 600.0):
+    def __init__(self, host: str, port: int, timeout_s: float = 2400.0):
         self.host = host
         self.port = port
         self.timeout_s = timeout_s
@@ -105,46 +104,22 @@ def runner_ready(runner: dict[str, Any]) -> bool:
     return "RunnerReady" in runner
 
 
-def runner_failed(runner: dict[str, Any]) -> bool:
-    return "RunnerFailed" in runner
-
-
-def get_runner_failed_message(runner: dict[str, Any]) -> str | None:
-    if "RunnerFailed" in runner:
-        return runner["RunnerFailed"].get("errorMessage")
-    return None
-
-
 def wait_for_instance_ready(
     client: ExoClient, instance_id: str, timeout: float = 24000.0
 ) -> None:
     start_time = time.time()
-    instance_existed = False
     while time.time() - start_time < timeout:
         state = client.request_json("GET", "/state")
         instances = state.get("instances", {})
 
         if instance_id not in instances:
-            if instance_existed:
-                # Instance was deleted after being created - likely due to runner failure
-                raise RuntimeError(
-                    f"Instance {instance_id} was deleted (runner may have failed)"
-                )
             time.sleep(0.1)
             continue
 
-        instance_existed = True
         instance = instances[instance_id]
         runner_ids = runner_ids_from_instance(instance)
         runners = state.get("runners", {})
 
-        # Check for failed runners first
-        for rid in runner_ids:
-            runner = runners.get(rid, {})
-            if runner_failed(runner):
-                error_msg = get_runner_failed_message(runner) or "Unknown error"
-                raise RuntimeError(f"Runner {rid} failed: {error_msg}")
-
         if all(runner_ready(runners.get(rid, {})) for rid in runner_ids):
             return
 
@@ -266,9 +241,6 @@ class PromptSizer:
             ids = tokenizer.apply_chat_template(
                 messages, tokenize=True, add_generation_prompt=True
             )
-            # Fix for transformers 5.x
-            if hasattr(ids, "input_ids"):
-                ids = ids.input_ids
             return int(len(ids))
 
         return count_fn
@@ -324,12 +296,6 @@ def main() -> int:
         default=4,
         help="Only consider placements using <= this many nodes.",
     )
-    ap.add_argument(
-        "--min-nodes",
-        type=int,
-        default=1,
-        help="Only consider placements using >= this many nodes.",
-    )
     ap.add_argument(
         "--instance-meta", choices=["ring", "jaccl", "both"], default="both"
     )
@@ -351,7 +317,7 @@ def main() -> int:
         help="Warmup runs per placement (uses first pp/tg).",
     )
     ap.add_argument(
-        "--timeout", type=float, default=600.0, help="HTTP timeout (seconds)."
+        "--timeout", type=float, default=2400.0, help="HTTP timeout (seconds)."
     )
     ap.add_argument(
         "--json-out",
@@ -430,7 +396,7 @@ def main() -> int:
         ):
             continue
 
-        if args.min_nodes <= n <= args.max_nodes:
+        if 0 < n <= args.max_nodes:
             selected.append(p)
 
     if not selected:
@@ -472,13 +438,7 @@ def main() -> int:
         )
 
         client.request_json("POST", "/instance", body={"instance": instance})
-        try:
-            wait_for_instance_ready(client, instance_id)
-        except (RuntimeError, TimeoutError) as e:
-            logger.error(f"Failed to initialize placement: {e}")
-            with contextlib.suppress(ExoHttpError):
-                client.request_json("DELETE", f"/instance/{instance_id}")
-            continue
+        wait_for_instance_ready(client, instance_id)
 
         time.sleep(1)
 

dashboard/src/lib/components/ChatForm.svelte

@@ -60,39 +60,12 @@
 		return models;
 	});
 
-	// Track previous model IDs to detect newly added models (plain variable to avoid reactive loop)
-	let previousModelIds: Set<string> = new Set();
-
-	// Auto-select the first available model if none is selected, if current selection is stale, or if a new model is added
+	// Auto-select the first available model if none is selected
 	$effect(() => {
 		const models = availableModels();
-		const currentModelIds = new Set(models.map(m => m.id));
-
-		if (models.length > 0) {
-			// Find newly added models (in current but not in previous)
-			const newModels = models.filter(m => !previousModelIds.has(m.id));
-
-			// If no model selected, select the first available
-			if (!currentModel) {
-				setSelectedChatModel(models[0].id);
-			}
-			// If current model is stale (no longer has a running instance), reset to first available
-			else if (!models.some(m => m.id === currentModel)) {
-				setSelectedChatModel(models[0].id);
-			}
-			// If a new model was just added, select it
-			else if (newModels.length > 0 && previousModelIds.size > 0) {
-				setSelectedChatModel(newModels[0].id);
-			}
-		} else {
-			// No instances running - clear the selected model
-			if (currentModel) {
-				setSelectedChatModel('');
-			}
+		if (models.length > 0 && !currentModel) {
+			setSelectedChatModel(models[0].id);
 		}
-
-		// Update previous model IDs for next comparison
-		previousModelIds = currentModelIds;
 	});
 
 	function getInstanceModelId(instanceWrapped: unknown): string {

dashboard/src/routes/+page.svelte

@@ -400,8 +400,10 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 				const errorText = await response.text();
 				console.error('Failed to launch instance:', errorText);
 			} else {
-				// Always auto-select the newly launched model so the user chats to what they just launched
-				setSelectedChatModel(modelId);
+				// Auto-select the launched model only if no model is currently selected
+				if (!selectedChatModel()) {
+					setSelectedChatModel(modelId);
+				}
 				
 				// Scroll to the bottom of instances container to show the new instance
 				// Use multiple attempts to ensure DOM has updated with the new instance
@@ -761,10 +763,6 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 	async function deleteInstance(instanceId: string) {
 		if (!confirm(`Delete instance ${instanceId.slice(0, 8)}...?`)) return;
 		
-		// Get the model ID of the instance being deleted before we delete it
-		const deletedInstanceModelId = getInstanceModelId(instanceData[instanceId]);
-		const wasSelected = selectedChatModel() === deletedInstanceModelId;
-		
 		try {
 			const response = await fetch(`/instance/${instanceId}`, {
 				method: 'DELETE',
@@ -773,24 +771,6 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 			
 			if (!response.ok) {
 				console.error('Failed to delete instance:', response.status);
-			} else if (wasSelected) {
-				// If we deleted the currently selected model, switch to another available model
-				// Find another instance that isn't the one we just deleted
-				const remainingInstances = Object.entries(instanceData).filter(([id]) => id !== instanceId);
-				if (remainingInstances.length > 0) {
-					// Select the last instance (most recently added, since objects preserve insertion order)
-					const [, lastInstance] = remainingInstances[remainingInstances.length - 1];
-					const newModelId = getInstanceModelId(lastInstance);
-					if (newModelId && newModelId !== 'Unknown' && newModelId !== 'Unknown Model') {
-						setSelectedChatModel(newModelId);
-					} else {
-						// Clear selection if no valid model found
-						setSelectedChatModel('');
-					}
-				} else {
-					// No more instances, clear the selection
-					setSelectedChatModel('');
-				}
 			}
 		} catch (error) {
 			console.error('Error deleting instance:', error);

justfile

@@ -1,5 +1,3 @@
-export NIX_CONFIG := "extra-experimental-features = nix-command flakes"
-
 fmt:
     nix fmt
 

pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
     "tiktoken>=0.12.0", # required for kimi k2 tokenizer
     "hypercorn>=0.18.0",
     "openai-harmony>=0.0.8",
-    "httpx>=0.28.1",
+    "tomlkit>=0.14.0",
 ]
 
 [project.scripts]

resources/model_cards/deepseek-v3.1-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "deepseek-v3.1-4bit"
+model_id = "mlx-community/DeepSeek-V3.1-4bit"
+name = "DeepSeek V3.1 (4-bit)"
+description = "DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/DeepSeek-V3.1-4bit"
+pretty_name = "DeepSeek V3.1 (4-bit)"
+n_layers = 61
+hidden_size = 7168
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 405874409472

resources/model_cards/deepseek-v3.1-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "deepseek-v3.1-8bit"
+model_id = "mlx-community/DeepSeek-V3.1-8bit"
+name = "DeepSeek V3.1 (8-bit)"
+description = "DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/DeepSeek-V3.1-8bit"
+pretty_name = "DeepSeek V3.1 (8-bit)"
+n_layers = 61
+hidden_size = 7168
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 765577920512

resources/model_cards/glm-4.5-air-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "glm-4.5-air-8bit"
+model_id = "mlx-community/GLM-4.5-Air-8bit"
+name = "GLM 4.5 Air 8bit"
+description = "GLM 4.5 Air 8bit"
+tags = []
+
+[metadata]
+model_id = "mlx-community/GLM-4.5-Air-8bit"
+pretty_name = "GLM 4.5 Air 8bit"
+n_layers = 46
+hidden_size = 4096
+supports_tensor = false
+
+[metadata.storage_size]
+in_bytes = 122406567936

resources/model_cards/glm-4.5-air-bf16.toml

@@ -0,0 +1,15 @@
+short_id = "glm-4.5-air-bf16"
+model_id = "mlx-community/GLM-4.5-Air-bf16"
+name = "GLM 4.5 Air bf16"
+description = "GLM 4.5 Air bf16"
+tags = []
+
+[metadata]
+model_id = "mlx-community/GLM-4.5-Air-bf16"
+pretty_name = "GLM 4.5 Air bf16"
+n_layers = 46
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 229780750336

resources/model_cards/glm-4.7-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "glm-4.7-4bit"
+model_id = "mlx-community/GLM-4.7-4bit"
+name = "GLM 4.7 4bit"
+description = "GLM 4.7 4bit"
+tags = []
+
+[metadata]
+model_id = "mlx-community/GLM-4.7-4bit"
+pretty_name = "GLM 4.7 4bit"
+n_layers = 91
+hidden_size = 5120
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 198556925568

resources/model_cards/glm-4.7-6bit.toml

@@ -0,0 +1,15 @@
+short_id = "glm-4.7-6bit"
+model_id = "mlx-community/GLM-4.7-6bit"
+name = "GLM 4.7 6bit"
+description = "GLM 4.7 6bit"
+tags = []
+
+[metadata]
+model_id = "mlx-community/GLM-4.7-6bit"
+pretty_name = "GLM 4.7 6bit"
+n_layers = 91
+hidden_size = 5120
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 286737579648

resources/model_cards/glm-4.7-8bit-gs32.toml

@@ -0,0 +1,15 @@
+short_id = "glm-4.7-8bit-gs32"
+model_id = "mlx-community/GLM-4.7-8bit-gs32"
+name = "GLM 4.7 8bit (gs32)"
+description = "GLM 4.7 8bit (gs32)"
+tags = []
+
+[metadata]
+model_id = "mlx-community/GLM-4.7-8bit-gs32"
+pretty_name = "GLM 4.7 8bit (gs32)"
+n_layers = 91
+hidden_size = 5120
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 396963397248

resources/model_cards/gpt-oss-120b-MXFP4-Q8.toml

@@ -0,0 +1,15 @@
+short_id = "gpt-oss-120b-MXFP4-Q8"
+model_id = "mlx-community/gpt-oss-120b-MXFP4-Q8"
+name = "GPT-OSS 120B (MXFP4-Q8, MLX)"
+description = "OpenAI's GPT-OSS 120B is a 117B-parameter Mixture-of-Experts model designed for high-reasoning and general-purpose use; this variant is a 4-bit MLX conversion for Apple Silicon."
+tags = []
+
+[metadata]
+model_id = "mlx-community/gpt-oss-120b-MXFP4-Q8"
+pretty_name = "GPT-OSS 120B (MXFP4-Q8, MLX)"
+n_layers = 36
+hidden_size = 2880
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 70652212224

resources/model_cards/gpt-oss-20b-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "gpt-oss-20b-4bit"
+model_id = "mlx-community/gpt-oss-20b-MXFP4-Q4"
+name = "GPT-OSS 20B (MXFP4-Q4, MLX)"
+description = "OpenAI's GPT-OSS 20B is a medium-sized MoE model for lower-latency and local or specialized use cases; this MLX variant uses MXFP4 4-bit quantization."
+tags = []
+
+[metadata]
+model_id = "mlx-community/gpt-oss-20b-MXFP4-Q4"
+pretty_name = "GPT-OSS 20B (MXFP4-Q4, MLX)"
+n_layers = 24
+hidden_size = 2880
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 12025908224

resources/model_cards/kimi-k2-instruct-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "kimi-k2-instruct-4bit"
+model_id = "mlx-community/Kimi-K2-Instruct-4bit"
+name = "Kimi K2 Instruct (4-bit)"
+description = "Kimi K2 is a large language model trained on the Kimi K2 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Kimi-K2-Instruct-4bit"
+pretty_name = "Kimi K2 Instruct (4-bit)"
+n_layers = 61
+hidden_size = 7168
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 620622774272

resources/model_cards/kimi-k2-thinking.toml

@@ -0,0 +1,15 @@
+short_id = "kimi-k2-thinking"
+model_id = "mlx-community/Kimi-K2-Thinking"
+name = "Kimi K2 Thinking (4-bit)"
+description = "Kimi K2 Thinking is the latest, most capable version of open-source thinking model."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Kimi-K2-Thinking"
+pretty_name = "Kimi K2 Thinking (4-bit)"
+n_layers = 61
+hidden_size = 7168
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 706522120192

resources/model_cards/llama-3.1-70b.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.1-70b"
+model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
+name = "Llama 3.1 70B (4-bit)"
+description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
+pretty_name = "Llama 3.1 70B (4-bit)"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 40652242944

resources/model_cards/llama-3.1-8b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.1-8b-8bit"
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
+name = "Llama 3.1 8B (8-bit)"
+description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
+pretty_name = "Llama 3.1 8B (8-bit)"
+n_layers = 32
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 8954839040

resources/model_cards/llama-3.1-8b-bf16.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.1-8b-bf16"
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
+name = "Llama 3.1 8B (BF16)"
+description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
+pretty_name = "Llama 3.1 8B (BF16)"
+n_layers = 32
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 16882073600

resources/model_cards/llama-3.1-8b.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.1-8b"
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
+name = "Llama 3.1 8B (4-bit)"
+description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
+pretty_name = "Llama 3.1 8B (4-bit)"
+n_layers = 32
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 4637851648

resources/model_cards/llama-3.2-1b.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.2-1b"
+model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
+name = "Llama 3.2 1B (4-bit)"
+description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
+pretty_name = "Llama 3.2 1B (4-bit)"
+n_layers = 16
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 729808896

resources/model_cards/llama-3.2-3b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.2-3b-8bit"
+model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
+name = "Llama 3.2 3B (8-bit)"
+description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
+pretty_name = "Llama 3.2 3B (8-bit)"
+n_layers = 28
+hidden_size = 3072
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 3501195264

resources/model_cards/llama-3.2-3b.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.2-3b"
+model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
+name = "Llama 3.2 3B (4-bit)"
+description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
+pretty_name = "Llama 3.2 3B (4-bit)"
+n_layers = 28
+hidden_size = 3072
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 1863319552

resources/model_cards/llama-3.3-70b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.3-70b-8bit"
+model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
+name = "Llama 3.3 70B (8-bit)"
+description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
+pretty_name = "Llama 3.3 70B (8-bit)"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 76799803392

resources/model_cards/llama-3.3-70b-fp16.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.3-70b-fp16"
+model_id = "mlx-community/llama-3.3-70b-instruct-fp16"
+name = "Llama 3.3 70B (FP16)"
+description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
+tags = []
+
+[metadata]
+model_id = "mlx-community/llama-3.3-70b-instruct-fp16"
+pretty_name = "Llama 3.3 70B (FP16)"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 144383672320

resources/model_cards/llama-3.3-70b.toml

@@ -0,0 +1,15 @@
+short_id = "llama-3.3-70b"
+model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
+name = "Llama 3.3 70B (4-bit)"
+description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
+pretty_name = "Llama 3.3 70B"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 40652242944

resources/model_cards/minimax-m2.1-3bit.toml

@@ -0,0 +1,15 @@
+short_id = "minimax-m2.1-3bit"
+model_id = "mlx-community/MiniMax-M2.1-3bit"
+name = "MiniMax M2.1 3bit"
+description = "MiniMax M2.1 3bit"
+tags = []
+
+[metadata]
+model_id = "mlx-community/MiniMax-M2.1-3bit"
+pretty_name = "MiniMax M2.1 3bit"
+n_layers = 61
+hidden_size = 3072
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 100086644736

resources/model_cards/minimax-m2.1-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "minimax-m2.1-8bit"
+model_id = "mlx-community/MiniMax-M2.1-8bit"
+name = "MiniMax M2.1 8bit"
+description = "MiniMax M2.1 8bit"
+tags = []
+
+[metadata]
+model_id = "mlx-community/MiniMax-M2.1-8bit"
+pretty_name = "MiniMax M2.1 8bit"
+n_layers = 61
+hidden_size = 3072
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 242986745856

resources/model_cards/qwen3-0.6b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-0.6b-8bit"
+model_id = "mlx-community/Qwen3-0.6B-8bit"
+name = "Qwen3 0.6B (8-bit)"
+description = "Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-0.6B-8bit"
+pretty_name = "Qwen3 0.6B (8-bit)"
+n_layers = 28
+hidden_size = 1024
+supports_tensor = false
+
+[metadata.storage_size]
+in_bytes = 698351616

resources/model_cards/qwen3-0.6b.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-0.6b"
+model_id = "mlx-community/Qwen3-0.6B-4bit"
+name = "Qwen3 0.6B (4-bit)"
+description = "Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-0.6B-4bit"
+pretty_name = "Qwen3 0.6B (4-bit)"
+n_layers = 28
+hidden_size = 1024
+supports_tensor = false
+
+[metadata.storage_size]
+in_bytes = 342884352

resources/model_cards/qwen3-235b-a22b-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-235b-a22b-4bit"
+model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
+name = "Qwen3 235B A22B (4-bit)"
+description = "Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
+pretty_name = "Qwen3 235B A22B (4-bit)"
+n_layers = 94
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 141733920768

resources/model_cards/qwen3-235b-a22b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-235b-a22b-8bit"
+model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
+name = "Qwen3 235B A22B (8-bit)"
+description = "Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
+pretty_name = "Qwen3 235B A22B (8-bit)"
+n_layers = 94
+hidden_size = 4096
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 268435456000

resources/model_cards/qwen3-30b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-30b-8bit"
+model_id = "mlx-community/Qwen3-30B-A3B-8bit"
+name = "Qwen3 30B A3B (8-bit)"
+description = "Qwen3 30B is a large language model trained on the Qwen3 30B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-30B-A3B-8bit"
+pretty_name = "Qwen3 30B A3B (8-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 33279705088

resources/model_cards/qwen3-30b.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-30b"
+model_id = "mlx-community/Qwen3-30B-A3B-4bit"
+name = "Qwen3 30B A3B (4-bit)"
+description = "Qwen3 30B is a large language model trained on the Qwen3 30B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-30B-A3B-4bit"
+pretty_name = "Qwen3 30B A3B (4-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 17612931072

resources/model_cards/qwen3-80b-a3B-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-80b-a3B-4bit"
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
+name = "Qwen3 80B A3B (4-bit)"
+description = "Qwen3 80B"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
+pretty_name = "Qwen3 80B A3B (4-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 46976204800

resources/model_cards/qwen3-80b-a3B-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-80b-a3B-8bit"
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
+name = "Qwen3 80B A3B (8-bit)"
+description = "Qwen3 80B"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
+pretty_name = "Qwen3 80B A3B (8-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 88814387200

resources/model_cards/qwen3-80b-a3B-thinking-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-80b-a3B-thinking-4bit"
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
+name = "Qwen3 80B A3B Thinking (4-bit)"
+description = "Qwen3 80B Reasoning model"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
+pretty_name = "Qwen3 80B A3B (4-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 88814387200

resources/model_cards/qwen3-80b-a3B-thinking-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-80b-a3B-thinking-8bit"
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
+name = "Qwen3 80B A3B Thinking (8-bit)"
+description = "Qwen3 80B Reasoning model"
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
+pretty_name = "Qwen3 80B A3B (8-bit)"
+n_layers = 48
+hidden_size = 2048
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 88814387200

resources/model_cards/qwen3-coder-480b-a35b-4bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-coder-480b-a35b-4bit"
+model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
+name = "Qwen3 Coder 480B A35B (4-bit)"
+description = "Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
+pretty_name = "Qwen3 Coder 480B A35B (4-bit)"
+n_layers = 62
+hidden_size = 6144
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 289910292480

resources/model_cards/qwen3-coder-480b-a35b-8bit.toml

@@ -0,0 +1,15 @@
+short_id = "qwen3-coder-480b-a35b-8bit"
+model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
+name = "Qwen3 Coder 480B A35B (8-bit)"
+description = "Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset."
+tags = []
+
+[metadata]
+model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
+pretty_name = "Qwen3 Coder 480B A35B (8-bit)"
+n_layers = 62
+hidden_size = 6144
+supports_tensor = true
+
+[metadata.storage_size]
+in_bytes = 579820584960

src/exo/main.py

@@ -205,14 +205,6 @@ def main():
     logger.info("Starting EXO")
     logger.info(f"EXO_LIBP2P_NAMESPACE: {os.getenv('EXO_LIBP2P_NAMESPACE')}")
 
-    # Set FAST_SYNCH override env var for runner subprocesses
-    if args.fast_synch is True:
-        os.environ["EXO_FAST_SYNCH"] = "on"
-        logger.info("FAST_SYNCH forced ON")
-    elif args.fast_synch is False:
-        os.environ["EXO_FAST_SYNCH"] = "off"
-        logger.info("FAST_SYNCH forced OFF")
-
     node = anyio.run(Node.create, args)
     anyio.run(node.run)
     logger.info("EXO Shutdown complete")
@@ -226,7 +218,6 @@ class Args(CamelCaseModel):
     api_port: PositiveInt = 52415
     tb_only: bool = False
     no_worker: bool = False
-    fast_synch: bool | None = None  # None = auto, True = force on, False = force off
 
     @classmethod
     def parse(cls) -> Self:
@@ -268,20 +259,6 @@ class Args(CamelCaseModel):
             "--no-worker",
             action="store_true",
         )
-        fast_synch_group = parser.add_mutually_exclusive_group()
-        fast_synch_group.add_argument(
-            "--fast-synch",
-            action="store_true",
-            dest="fast_synch",
-            default=None,
-            help="Force MLX FAST_SYNCH on (for JACCL backend)",
-        )
-        fast_synch_group.add_argument(
-            "--no-fast-synch",
-            action="store_false",
-            dest="fast_synch",
-            help="Force MLX FAST_SYNCH off",
-        )
 
         args = parser.parse_args()
         return cls(**vars(args))  # pyright: ignore[reportAny] - We are intentionally validating here, we can't do it statically

src/exo/master/api.py

@@ -1,19 +1,24 @@
 import time
 from collections.abc import AsyncGenerator
-from http import HTTPStatus
 from typing import cast
 
 import anyio
-from anyio import BrokenResourceError, create_task_group
+from anyio import create_task_group
 from anyio.abc import TaskGroup
-from fastapi import FastAPI, HTTPException, Request
+from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from hypercorn.asyncio import serve  # pyright: ignore[reportUnknownVariableType]
 from hypercorn.config import Config
 from hypercorn.typing import ASGIFramework
 from loguru import logger
+from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
+    HarmonyEncodingName,
+    Role,
+    StreamableParser,
+    load_harmony_encoding,
+)
 
 from exo.master.placement import place_instance as get_instance_placements
 from exo.shared.apply import apply
@@ -30,8 +35,6 @@ from exo.shared.types.api import (
     CreateInstanceParams,
     CreateInstanceResponse,
     DeleteInstanceResponse,
-    ErrorInfo,
-    ErrorResponse,
     FinishReason,
     GenerationStats,
     ModelList,
@@ -52,12 +55,7 @@ from exo.shared.types.commands import (
     TaskFinished,
 )
 from exo.shared.types.common import CommandId, NodeId, SessionId
-from exo.shared.types.events import (
-    ChunkGenerated,
-    Event,
-    ForwarderEvent,
-    IndexedEvent,
-)
+from exo.shared.types.events import ChunkGenerated, Event, ForwarderEvent, IndexedEvent
 from exo.shared.types.memory import Memory
 from exo.shared.types.models import ModelId, ModelMetadata
 from exo.shared.types.state import State
@@ -69,6 +67,8 @@ from exo.utils.channels import Receiver, Sender, channel
 from exo.utils.dashboard_path import find_dashboard
 from exo.utils.event_buffer import OrderedBuffer
 
+encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
+
 
 def chunk_to_response(
     chunk: TokenChunk, command_id: CommandId
@@ -123,7 +123,6 @@ class API:
         self.paused_ev: anyio.Event = anyio.Event()
 
         self.app = FastAPI()
-        self._setup_exception_handlers()
         self._setup_cors()
         self._setup_routes()
 
@@ -154,20 +153,6 @@ class API:
         self.paused_ev.set()
         self.paused_ev = anyio.Event()
 
-    def _setup_exception_handlers(self) -> None:
-        @self.app.exception_handler(HTTPException)
-        async def http_exception_handler(  # pyright: ignore[reportUnusedFunction]
-            _: Request, exc: HTTPException
-        ) -> JSONResponse:
-            err = ErrorResponse(
-                error=ErrorInfo(
-                    message=exc.detail,
-                    type=HTTPStatus(exc.status_code).phrase,
-                    code=exc.status_code,
-                )
-            )
-            return JSONResponse(err.model_dump(), status_code=exc.status_code)
-
     def _setup_cors(self) -> None:
         self.app.add_middleware(
             CORSMiddleware,
@@ -396,8 +381,35 @@ class API:
             instance_id=instance_id,
         )
 
+    async def _process_gpt_oss(self, token_chunks: Receiver[TokenChunk]):
+        stream = StreamableParser(encoding, role=Role.ASSISTANT)
+        thinking = False
+
+        async for chunk in token_chunks:
+            stream.process(chunk.token_id)
+
+            delta = stream.last_content_delta
+            ch = stream.current_channel
+
+            if ch == "analysis" and not thinking:
+                thinking = True
+                yield chunk.model_copy(update={"text": "<think>"})
+
+            if ch != "analysis" and thinking:
+                thinking = False
+                yield chunk.model_copy(update={"text": "</think>"})
+
+            if delta:
+                yield chunk.model_copy(update={"text": delta})
+
+            if chunk.finish_reason is not None:
+                if thinking:
+                    yield chunk.model_copy(update={"text": "</think>"})
+                yield chunk
+                break
+
     async def _chat_chunk_stream(
-        self, command_id: CommandId
+        self, command_id: CommandId, parse_gpt_oss: bool
     ) -> AsyncGenerator[TokenChunk, None]:
         """Yield `TokenChunk`s for a given command until completion."""
 
@@ -405,10 +417,16 @@ class API:
             self._chat_completion_queues[command_id], recv = channel[TokenChunk]()
 
             with recv as token_chunks:
-                async for chunk in token_chunks:
-                    yield chunk
-                    if chunk.finish_reason is not None:
-                        break
+                if parse_gpt_oss:
+                    async for chunk in self._process_gpt_oss(token_chunks):
+                        yield chunk
+                        if chunk.finish_reason is not None:
+                            break
+                else:
+                    async for chunk in token_chunks:
+                        yield chunk
+                        if chunk.finish_reason is not None:
+                            break
 
         except anyio.get_cancelled_exc_class():
             # TODO: TaskCancelled
@@ -424,23 +442,11 @@ class API:
             del self._chat_completion_queues[command_id]
 
     async def _generate_chat_stream(
-        self, command_id: CommandId
+        self, command_id: CommandId, parse_gpt_oss: bool
     ) -> AsyncGenerator[str, None]:
         """Generate chat completion stream as JSON strings."""
 
-        async for chunk in self._chat_chunk_stream(command_id):
-            if chunk.finish_reason == "error":
-                error_response = ErrorResponse(
-                    error=ErrorInfo(
-                        message=chunk.error_message or "Internal server error",
-                        type="InternalServerError",
-                        code=500,
-                    )
-                )
-                yield f"data: {error_response.model_dump_json()}\n\n"
-                yield "data: [DONE]\n\n"
-                return
-
+        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
             chunk_response: ChatCompletionResponse = chunk_to_response(
                 chunk, command_id
             )
@@ -452,7 +458,7 @@ class API:
                 yield "data: [DONE]\n\n"
 
     async def _collect_chat_completion(
-        self, command_id: CommandId
+        self, command_id: CommandId, parse_gpt_oss: bool
     ) -> ChatCompletionResponse:
         """Collect all token chunks for a chat completion and return a single response."""
 
@@ -460,13 +466,7 @@ class API:
         model: str | None = None
         finish_reason: FinishReason | None = None
 
-        async for chunk in self._chat_chunk_stream(command_id):
-            if chunk.finish_reason == "error":
-                raise HTTPException(
-                    status_code=500,
-                    detail=chunk.error_message or "Internal server error",
-                )
-
+        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
             if model is None:
                 model = chunk.model
 
@@ -495,7 +495,7 @@ class API:
         )
 
     async def _collect_chat_completion_with_stats(
-        self, command_id: CommandId
+        self, command_id: CommandId, parse_gpt_oss: bool
     ) -> BenchChatCompletionResponse:
         text_parts: list[str] = []
         model: str | None = None
@@ -503,13 +503,7 @@ class API:
 
         stats: GenerationStats | None = None
 
-        async for chunk in self._chat_chunk_stream(command_id):
-            if chunk.finish_reason == "error":
-                raise HTTPException(
-                    status_code=500,
-                    detail=chunk.error_message or "Internal server error",
-                )
-
+        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
             if model is None:
                 model = chunk.model
 
@@ -550,6 +544,8 @@ class API:
         """Handle chat completions, supporting both streaming and non-streaming responses."""
         model_meta = await resolve_model_meta(payload.model)
         payload.model = model_meta.model_id
+        parse_gpt_oss = "gpt-oss" in model_meta.model_id.lower()
+        logger.info(f"{parse_gpt_oss=}")
 
         if not any(
             instance.shard_assignments.model_id == payload.model
@@ -566,16 +562,17 @@ class API:
         await self._send(command)
         if payload.stream:
             return StreamingResponse(
-                self._generate_chat_stream(command.command_id),
+                self._generate_chat_stream(command.command_id, parse_gpt_oss),
                 media_type="text/event-stream",
             )
 
-        return await self._collect_chat_completion(command.command_id)
+        return await self._collect_chat_completion(command.command_id, parse_gpt_oss)
 
     async def bench_chat_completions(
         self, payload: BenchChatCompletionTaskParams
     ) -> BenchChatCompletionResponse:
         model_meta = await resolve_model_meta(payload.model)
+        parse_gpt_oss = "gpt-oss" in model_meta.model_id.lower()
         payload.model = model_meta.model_id
 
         if not any(
@@ -592,7 +589,10 @@ class API:
         command = ChatCompletion(request_params=payload)
         await self._send(command)
 
-        response = await self._collect_chat_completion_with_stats(command.command_id)
+        response = await self._collect_chat_completion_with_stats(
+            command.command_id,
+            parse_gpt_oss,
+        )
         return response
 
     def _calculate_total_available_memory(self) -> Memory:
@@ -654,14 +654,14 @@ class API:
                 for idx, event in self.event_buffer.drain_indexed():
                     self._event_log.append(event)
                     self.state = apply(self.state, IndexedEvent(event=event, idx=idx))
-                    if isinstance(event, ChunkGenerated):
+                    if (
+                        isinstance(event, ChunkGenerated)
+                        and event.command_id in self._chat_completion_queues
+                    ):
                         assert isinstance(event.chunk, TokenChunk)
-                        queue = self._chat_completion_queues.get(event.command_id)
-                        if queue is not None:
-                            try:
-                                await queue.send(event.chunk)
-                            except BrokenResourceError:
-                                self._chat_completion_queues.pop(event.command_id, None)
+                        await self._chat_completion_queues[event.command_id].send(
+                            event.chunk
+                        )
 
     async def _pause_on_new_election(self):
         with self.election_receiver as ems:

src/exo/master/tests/test_api_error_handling.py

@@ -1,107 +0,0 @@
-# pyright: reportUnusedFunction=false, reportAny=false
-from typing import Any, get_args
-
-from fastapi import FastAPI, HTTPException
-from fastapi.testclient import TestClient
-
-from exo.shared.types.api import ErrorInfo, ErrorResponse, FinishReason
-from exo.shared.types.chunks import TokenChunk
-from exo.worker.tests.constants import MODEL_A_ID
-
-
-def test_http_exception_handler_formats_openai_style() -> None:
-    """Test that HTTPException is converted to OpenAI-style error format."""
-    from exo.master.api import API
-
-    app = FastAPI()
-
-    # Setup exception handler
-    api = object.__new__(API)
-    api.app = app
-    api._setup_exception_handlers()  # pyright: ignore[reportPrivateUsage]
-
-    # Add test routes that raise HTTPException
-    @app.get("/test-error")
-    async def _test_error() -> None:
-        raise HTTPException(status_code=500, detail="Test error message")
-
-    @app.get("/test-not-found")
-    async def _test_not_found() -> None:
-        raise HTTPException(status_code=404, detail="Resource not found")
-
-    client = TestClient(app)
-
-    # Test 500 error
-    response = client.get("/test-error")
-    assert response.status_code == 500
-    data: dict[str, Any] = response.json()
-    assert "error" in data
-    assert data["error"]["message"] == "Test error message"
-    assert data["error"]["type"] == "Internal Server Error"
-    assert data["error"]["code"] == 500
-
-    # Test 404 error
-    response = client.get("/test-not-found")
-    assert response.status_code == 404
-    data = response.json()
-    assert "error" in data
-    assert data["error"]["message"] == "Resource not found"
-    assert data["error"]["type"] == "Not Found"
-    assert data["error"]["code"] == 404
-
-
-def test_finish_reason_includes_error() -> None:
-    valid_reasons = get_args(FinishReason)
-    assert "error" in valid_reasons
-
-
-def test_token_chunk_with_error_fields() -> None:
-    chunk = TokenChunk(
-        idx=0,
-        model=MODEL_A_ID,
-        text="",
-        token_id=0,
-        finish_reason="error",
-        error_message="Something went wrong",
-    )
-
-    assert chunk.finish_reason == "error"
-    assert chunk.error_message == "Something went wrong"
-
-
-def test_token_chunk_without_error() -> None:
-    chunk = TokenChunk(
-        idx=1,
-        model=MODEL_A_ID,
-        text="Hello",
-        token_id=42,
-        finish_reason=None,
-    )
-
-    assert chunk.finish_reason is None
-    assert chunk.error_message is None
-
-
-def test_error_response_construction() -> None:
-    error_response = ErrorResponse(
-        error=ErrorInfo(
-            message="Generation failed",
-            type="InternalServerError",
-            code=500,
-        )
-    )
-
-    assert error_response.error.message == "Generation failed"
-    assert error_response.error.code == 500
-
-
-def test_normal_finish_reasons_still_work() -> None:
-    for reason in ["stop", "length", "tool_calls", "content_filter", "function_call"]:
-        chunk = TokenChunk(
-            idx=0,
-            model=MODEL_A_ID,
-            text="done",
-            token_id=100,
-            finish_reason=reason,  # type: ignore[arg-type]
-        )
-        assert chunk.finish_reason == reason

src/exo/shared/logging.py

@@ -29,11 +29,6 @@ class _InterceptHandler(logging.Handler):
 
 def logger_setup(log_file: Path | None, verbosity: int = 0):
     """Set up logging for this process - formatting, file handles, verbosity and output"""
-
-    logging.getLogger("exo_pyo3_bindings").setLevel(logging.WARNING)
-    logging.getLogger("httpx").setLevel(logging.WARNING)
-    logging.getLogger("httpcore").setLevel(logging.WARNING)
-
     logger.remove()
 
     # replace all stdlib loggers with _InterceptHandlers that log to loguru

src/exo/shared/models/model_cards.py

@@ -1,5 +1,8 @@
-from exo.shared.types.memory import Memory
+from anyio import Path, open_file
+import tomlkit
+
 from exo.shared.types.models import ModelId, ModelMetadata
+from exo.shared.models.model_meta import get_model_meta
 from exo.utils.pydantic_ext import CamelCaseModel
 
 
@@ -11,542 +14,27 @@ class ModelCard(CamelCaseModel):
     tags: list[str]
     metadata: ModelMetadata
 
+    @staticmethod
+    async def load(path: Path) -> "ModelCard":
+        async with await open_file(path) as f:
+            data = await f.read()
+            py = tomlkit.loads(data)
+            return ModelCard.model_validate(py)
 
-MODEL_CARDS: dict[str, ModelCard] = {
-    # deepseek v3
-    "deepseek-v3.1-4bit": ModelCard(
-        short_id="deepseek-v3.1-4bit",
-        model_id=ModelId("mlx-community/DeepSeek-V3.1-4bit"),
-        name="DeepSeek V3.1 (4-bit)",
-        description="""DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/DeepSeek-V3.1-4bit"),
-            pretty_name="DeepSeek V3.1 (4-bit)",
-            storage_size=Memory.from_gb(378),
-            n_layers=61,
-            hidden_size=7168,
-            supports_tensor=True,
-        ),
-    ),
-    "deepseek-v3.1-8bit": ModelCard(
-        short_id="deepseek-v3.1-8bit",
-        model_id=ModelId("mlx-community/DeepSeek-V3.1-8bit"),
-        name="DeepSeek V3.1 (8-bit)",
-        description="""DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/DeepSeek-V3.1-8bit"),
-            pretty_name="DeepSeek V3.1 (8-bit)",
-            storage_size=Memory.from_gb(713),
-            n_layers=61,
-            hidden_size=7168,
-            supports_tensor=True,
-        ),
-    ),
-    # kimi k2
-    "kimi-k2-instruct-4bit": ModelCard(
-        short_id="kimi-k2-instruct-4bit",
-        model_id=ModelId("mlx-community/Kimi-K2-Instruct-4bit"),
-        name="Kimi K2 Instruct (4-bit)",
-        description="""Kimi K2 is a large language model trained on the Kimi K2 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Kimi-K2-Instruct-4bit"),
-            pretty_name="Kimi K2 Instruct (4-bit)",
-            storage_size=Memory.from_gb(578),
-            n_layers=61,
-            hidden_size=7168,
-            supports_tensor=True,
-        ),
-    ),
-    "kimi-k2-thinking": ModelCard(
-        short_id="kimi-k2-thinking",
-        model_id=ModelId("mlx-community/Kimi-K2-Thinking"),
-        name="Kimi K2 Thinking (4-bit)",
-        description="""Kimi K2 Thinking is the latest, most capable version of open-source thinking model.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Kimi-K2-Thinking"),
-            pretty_name="Kimi K2 Thinking (4-bit)",
-            storage_size=Memory.from_gb(658),
-            n_layers=61,
-            hidden_size=7168,
-            supports_tensor=True,
-        ),
-    ),
-    # llama-3.1
-    "llama-3.1-8b": ModelCard(
-        short_id="llama-3.1-8b",
-        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"),
-        name="Llama 3.1 8B (4-bit)",
-        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"),
-            pretty_name="Llama 3.1 8B (4-bit)",
-            storage_size=Memory.from_mb(4423),
-            n_layers=32,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.1-8b-8bit": ModelCard(
-        short_id="llama-3.1-8b-8bit",
-        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"),
-        name="Llama 3.1 8B (8-bit)",
-        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"),
-            pretty_name="Llama 3.1 8B (8-bit)",
-            storage_size=Memory.from_mb(8540),
-            n_layers=32,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.1-8b-bf16": ModelCard(
-        short_id="llama-3.1-8b-bf16",
-        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"),
-        name="Llama 3.1 8B (BF16)",
-        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"),
-            pretty_name="Llama 3.1 8B (BF16)",
-            storage_size=Memory.from_mb(16100),
-            n_layers=32,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.1-70b": ModelCard(
-        short_id="llama-3.1-70b",
-        model_id=ModelId("mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"),
-        name="Llama 3.1 70B (4-bit)",
-        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"),
-            pretty_name="Llama 3.1 70B (4-bit)",
-            storage_size=Memory.from_mb(38769),
-            n_layers=80,
-            hidden_size=8192,
-            supports_tensor=True,
-        ),
-    ),
-    # llama-3.2
-    "llama-3.2-1b": ModelCard(
-        short_id="llama-3.2-1b",
-        model_id=ModelId("mlx-community/Llama-3.2-1B-Instruct-4bit"),
-        name="Llama 3.2 1B (4-bit)",
-        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Llama-3.2-1B-Instruct-4bit"),
-            pretty_name="Llama 3.2 1B (4-bit)",
-            storage_size=Memory.from_mb(696),
-            n_layers=16,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.2-3b": ModelCard(
-        short_id="llama-3.2-3b",
-        model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-4bit"),
-        name="Llama 3.2 3B (4-bit)",
-        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-4bit"),
-            pretty_name="Llama 3.2 3B (4-bit)",
-            storage_size=Memory.from_mb(1777),
-            n_layers=28,
-            hidden_size=3072,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.2-3b-8bit": ModelCard(
-        short_id="llama-3.2-3b-8bit",
-        model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-8bit"),
-        name="Llama 3.2 3B (8-bit)",
-        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-8bit"),
-            pretty_name="Llama 3.2 3B (8-bit)",
-            storage_size=Memory.from_mb(3339),
-            n_layers=28,
-            hidden_size=3072,
-            supports_tensor=True,
-        ),
-    ),
-    # llama-3.3
-    "llama-3.3-70b": ModelCard(
-        short_id="llama-3.3-70b",
-        model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-4bit"),
-        name="Llama 3.3 70B (4-bit)",
-        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-4bit"),
-            pretty_name="Llama 3.3 70B",
-            storage_size=Memory.from_mb(38769),
-            n_layers=80,
-            hidden_size=8192,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.3-70b-8bit": ModelCard(
-        short_id="llama-3.3-70b-8bit",
-        model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-8bit"),
-        name="Llama 3.3 70B (8-bit)",
-        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-8bit"),
-            pretty_name="Llama 3.3 70B (8-bit)",
-            storage_size=Memory.from_mb(73242),
-            n_layers=80,
-            hidden_size=8192,
-            supports_tensor=True,
-        ),
-    ),
-    "llama-3.3-70b-fp16": ModelCard(
-        short_id="llama-3.3-70b-fp16",
-        model_id=ModelId("mlx-community/llama-3.3-70b-instruct-fp16"),
-        name="Llama 3.3 70B (FP16)",
-        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/llama-3.3-70b-instruct-fp16"),
-            pretty_name="Llama 3.3 70B (FP16)",
-            storage_size=Memory.from_mb(137695),
-            n_layers=80,
-            hidden_size=8192,
-            supports_tensor=True,
-        ),
-    ),
-    # qwen3
-    "qwen3-0.6b": ModelCard(
-        short_id="qwen3-0.6b",
-        model_id=ModelId("mlx-community/Qwen3-0.6B-4bit"),
-        name="Qwen3 0.6B (4-bit)",
-        description="""Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-0.6B-4bit"),
-            pretty_name="Qwen3 0.6B (4-bit)",
-            storage_size=Memory.from_mb(327),
-            n_layers=28,
-            hidden_size=1024,
-            supports_tensor=False,
-        ),
-    ),
-    "qwen3-0.6b-8bit": ModelCard(
-        short_id="qwen3-0.6b-8bit",
-        model_id=ModelId("mlx-community/Qwen3-0.6B-8bit"),
-        name="Qwen3 0.6B (8-bit)",
-        description="""Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-0.6B-8bit"),
-            pretty_name="Qwen3 0.6B (8-bit)",
-            storage_size=Memory.from_mb(666),
-            n_layers=28,
-            hidden_size=1024,
-            supports_tensor=False,
-        ),
-    ),
-    "qwen3-30b": ModelCard(
-        short_id="qwen3-30b",
-        model_id=ModelId("mlx-community/Qwen3-30B-A3B-4bit"),
-        name="Qwen3 30B A3B (4-bit)",
-        description="""Qwen3 30B is a large language model trained on the Qwen3 30B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-30B-A3B-4bit"),
-            pretty_name="Qwen3 30B A3B (4-bit)",
-            storage_size=Memory.from_mb(16797),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-30b-8bit": ModelCard(
-        short_id="qwen3-30b-8bit",
-        model_id=ModelId("mlx-community/Qwen3-30B-A3B-8bit"),
-        name="Qwen3 30B A3B (8-bit)",
-        description="""Qwen3 30B is a large language model trained on the Qwen3 30B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-30B-A3B-8bit"),
-            pretty_name="Qwen3 30B A3B (8-bit)",
-            storage_size=Memory.from_mb(31738),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-80b-a3B-4bit": ModelCard(
-        short_id="qwen3-80b-a3B-4bit",
-        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"),
-        name="Qwen3 80B A3B (4-bit)",
-        description="""Qwen3 80B""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"),
-            pretty_name="Qwen3 80B A3B (4-bit)",
-            storage_size=Memory.from_mb(44800),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-80b-a3B-8bit": ModelCard(
-        short_id="qwen3-80b-a3B-8bit",
-        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"),
-        name="Qwen3 80B A3B (8-bit)",
-        description="""Qwen3 80B""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"),
-            pretty_name="Qwen3 80B A3B (8-bit)",
-            storage_size=Memory.from_mb(84700),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-80b-a3B-thinking-4bit": ModelCard(
-        short_id="qwen3-80b-a3B-thinking-4bit",
-        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"),
-        name="Qwen3 80B A3B Thinking (4-bit)",
-        description="""Qwen3 80B Reasoning model""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"),
-            pretty_name="Qwen3 80B A3B (4-bit)",
-            storage_size=Memory.from_mb(84700),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-80b-a3B-thinking-8bit": ModelCard(
-        short_id="qwen3-80b-a3B-thinking-8bit",
-        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"),
-        name="Qwen3 80B A3B Thinking (8-bit)",
-        description="""Qwen3 80B Reasoning model""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"),
-            pretty_name="Qwen3 80B A3B (8-bit)",
-            storage_size=Memory.from_mb(84700),
-            n_layers=48,
-            hidden_size=2048,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-235b-a22b-4bit": ModelCard(
-        short_id="qwen3-235b-a22b-4bit",
-        model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"),
-        name="Qwen3 235B A22B (4-bit)",
-        description="""Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"),
-            pretty_name="Qwen3 235B A22B (4-bit)",
-            storage_size=Memory.from_gb(132),
-            n_layers=94,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-235b-a22b-8bit": ModelCard(
-        short_id="qwen3-235b-a22b-8bit",
-        model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"),
-        name="Qwen3 235B A22B (8-bit)",
-        description="""Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"),
-            pretty_name="Qwen3 235B A22B (8-bit)",
-            storage_size=Memory.from_gb(250),
-            n_layers=94,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-coder-480b-a35b-4bit": ModelCard(
-        short_id="qwen3-coder-480b-a35b-4bit",
-        model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"),
-        name="Qwen3 Coder 480B A35B (4-bit)",
-        description="""Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"),
-            pretty_name="Qwen3 Coder 480B A35B (4-bit)",
-            storage_size=Memory.from_gb(270),
-            n_layers=62,
-            hidden_size=6144,
-            supports_tensor=True,
-        ),
-    ),
-    "qwen3-coder-480b-a35b-8bit": ModelCard(
-        short_id="qwen3-coder-480b-a35b-8bit",
-        model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"),
-        name="Qwen3 Coder 480B A35B (8-bit)",
-        description="""Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"),
-            pretty_name="Qwen3 Coder 480B A35B (8-bit)",
-            storage_size=Memory.from_gb(540),
-            n_layers=62,
-            hidden_size=6144,
-            supports_tensor=True,
-        ),
-    ),
-    # gpt-oss
-    "gpt-oss-120b-MXFP4-Q8": ModelCard(
-        short_id="gpt-oss-120b-MXFP4-Q8",
-        model_id=ModelId("mlx-community/gpt-oss-120b-MXFP4-Q8"),
-        name="GPT-OSS 120B (MXFP4-Q8, MLX)",
-        description="""OpenAI's GPT-OSS 120B is a 117B-parameter Mixture-of-Experts model designed for high-reasoning and general-purpose use; this variant is a 4-bit MLX conversion for Apple Silicon.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/gpt-oss-120b-MXFP4-Q8"),
-            pretty_name="GPT-OSS 120B (MXFP4-Q8, MLX)",
-            storage_size=Memory.from_kb(68_996_301),
-            n_layers=36,
-            hidden_size=2880,
-            supports_tensor=True,
-        ),
-    ),
-    "gpt-oss-20b-MXFP4-Q8": ModelCard(
-        short_id="gpt-oss-20b-MXFP4-Q8",
-        model_id=ModelId("mlx-community/gpt-oss-20b-MXFP4-Q8"),
-        name="GPT-OSS 20B (MXFP4-Q8, MLX)",
-        description="""OpenAI's GPT-OSS 20B is a medium-sized MoE model for lower-latency and local or specialized use cases; this variant is a 4-bit MLX conversion for Apple Silicon.""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/gpt-oss-20b-MXFP4-Q8"),
-            pretty_name="GPT-OSS 20B (MXFP4-Q8, MLX)",
-            storage_size=Memory.from_kb(11_744_051),
-            n_layers=24,
-            hidden_size=2880,
-            supports_tensor=True,
-        ),
-    ),
-    # glm 4.5
-    "glm-4.5-air-8bit": ModelCard(
-        # Needs to be quantized g32 or g16 to work with tensor parallel
-        short_id="glm-4.5-air-8bit",
-        model_id=ModelId("mlx-community/GLM-4.5-Air-8bit"),
-        name="GLM 4.5 Air 8bit",
-        description="""GLM 4.5 Air 8bit""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/GLM-4.5-Air-8bit"),
-            pretty_name="GLM 4.5 Air 8bit",
-            storage_size=Memory.from_gb(114),
-            n_layers=46,
-            hidden_size=4096,
-            supports_tensor=False,
-        ),
-    ),
-    "glm-4.5-air-bf16": ModelCard(
-        short_id="glm-4.5-air-bf16",
-        model_id=ModelId("mlx-community/GLM-4.5-Air-bf16"),
-        name="GLM 4.5 Air bf16",
-        description="""GLM 4.5 Air bf16""",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/GLM-4.5-Air-bf16"),
-            pretty_name="GLM 4.5 Air bf16",
-            storage_size=Memory.from_gb(214),
-            n_layers=46,
-            hidden_size=4096,
-            supports_tensor=True,
-        ),
-    ),
-    # glm 4.7
-    "glm-4.7-4bit": ModelCard(
-        short_id="glm-4.7-4bit",
-        model_id=ModelId("mlx-community/GLM-4.7-4bit"),
-        name="GLM 4.7 4bit",
-        description="GLM 4.7 4bit",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/GLM-4.7-4bit"),
-            pretty_name="GLM 4.7 4bit",
-            storage_size=Memory.from_bytes(198556925568),
-            n_layers=91,
-            hidden_size=5120,
-            supports_tensor=True,
-        ),
-    ),
-    "glm-4.7-6bit": ModelCard(
-        short_id="glm-4.7-6bit",
-        model_id=ModelId("mlx-community/GLM-4.7-6bit"),
-        name="GLM 4.7 6bit",
-        description="GLM 4.7 6bit",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/GLM-4.7-6bit"),
-            pretty_name="GLM 4.7 6bit",
-            storage_size=Memory.from_bytes(286737579648),
-            n_layers=91,
-            hidden_size=5120,
-            supports_tensor=True,
-        ),
-    ),
-    "glm-4.7-8bit-gs32": ModelCard(
-        short_id="glm-4.7-8bit-gs32",
-        model_id=ModelId("mlx-community/GLM-4.7-8bit-gs32"),
-        name="GLM 4.7 8bit (gs32)",
-        description="GLM 4.7 8bit (gs32)",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/GLM-4.7-8bit-gs32"),
-            pretty_name="GLM 4.7 8bit (gs32)",
-            storage_size=Memory.from_bytes(396963397248),
-            n_layers=91,
-            hidden_size=5120,
-            supports_tensor=True,
-        ),
-    ),
-    # minimax-m2
-    "minimax-m2.1-8bit": ModelCard(
-        short_id="minimax-m2.1-8bit",
-        model_id=ModelId("mlx-community/MiniMax-M2.1-8bit"),
-        name="MiniMax M2.1 8bit",
-        description="MiniMax M2.1 8bit",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/MiniMax-M2.1-8bit"),
-            pretty_name="MiniMax M2.1 8bit",
-            storage_size=Memory.from_bytes(242986745856),
-            n_layers=61,
-            hidden_size=3072,
-            supports_tensor=True,
-        ),
-    ),
-    "minimax-m2.1-3bit": ModelCard(
-        short_id="minimax-m2.1-3bit",
-        model_id=ModelId("mlx-community/MiniMax-M2.1-3bit"),
-        name="MiniMax M2.1 3bit",
-        description="MiniMax M2.1 3bit",
-        tags=[],
-        metadata=ModelMetadata(
-            model_id=ModelId("mlx-community/MiniMax-M2.1-3bit"),
-            pretty_name="MiniMax M2.1 3bit",
-            storage_size=Memory.from_bytes(100086644736),
-            n_layers=61,
-            hidden_size=3072,
-            supports_tensor=True,
-        ),
-    ),
-}
+    async def save(self, path: Path):
+        async with await open_file(path, "w") as f:
+            py = self.model_dump()
+            data = tomlkit.dumps(py)  # pyright: ignore[reportUnknownMemberType]
+            await f.write(data)
+
+    @staticmethod
+    async def from_hf(model_id: str) -> "ModelCard":
+        short_name = model_id.split("/")[-1]
+        return ModelCard(
+            short_id=short_name,
+            model_id=ModelId(model_id),
+            name=short_name,
+            description=f"Custom model from {model_id}",
+            tags=[],
+            metadata=await get_model_meta(model_id),
+        )

src/exo/shared/models/model_meta.py

@@ -6,7 +6,6 @@ from huggingface_hub import model_info
 from loguru import logger
 from pydantic import BaseModel, Field
 
-from exo.shared.models.model_cards import MODEL_CARDS
 from exo.shared.types.memory import Memory
 from exo.shared.types.models import ModelId, ModelMetadata
 from exo.worker.download.download_utils import (
@@ -108,19 +107,13 @@ async def _get_model_meta(model_id: str) -> ModelMetadata:
     config_data = await get_config_data(model_id)
     num_layers = config_data.layer_count
     mem_size_bytes = await get_safetensors_size(model_id)
-    model_card = next(
-        (card for card in MODEL_CARDS.values() if card.model_id == ModelId(model_id)),
-        None,
-    )
 
     return ModelMetadata(
         model_id=ModelId(model_id),
-        pretty_name=model_card.name if model_card is not None else model_id,
+        pretty_name=model_id,
         storage_size=mem_size_bytes,
         n_layers=num_layers,
         hidden_size=config_data.hidden_size or 0,
         # TODO: all custom models currently do not support tensor. We could add a dynamic test for this?
-        supports_tensor=model_card.metadata.supports_tensor
-        if model_card is not None
-        else False,
+        supports_tensor=False,
     )

src/exo/shared/types/api.py

@@ -11,21 +11,10 @@ from exo.shared.types.worker.instances import Instance, InstanceId, InstanceMeta
 from exo.shared.types.worker.shards import Sharding
 
 FinishReason = Literal[
-    "stop", "length", "tool_calls", "content_filter", "function_call", "error"
+    "stop", "length", "tool_calls", "content_filter", "function_call"
 ]
 
 
-class ErrorInfo(BaseModel):
-    message: str
-    type: str
-    param: str | None = None
-    code: int
-
-
-class ErrorResponse(BaseModel):
-    error: ErrorInfo
-
-
 class ModelListModel(BaseModel):
     id: str
     object: str = "model"

src/exo/shared/types/chunks.py

@@ -22,7 +22,6 @@ class TokenChunk(BaseChunk):
     token_id: int
     finish_reason: FinishReason | None = None
     stats: GenerationStats | None = None
-    error_message: str | None = None
 
 
 class ImageChunk(BaseChunk):

src/exo/worker/engines/mlx/constants.py

@@ -13,8 +13,3 @@ KV_CACHE_BITS: int | None = None
 
 # TODO: We should really make this opt-in, but Kimi requires trust_remote_code=True
 TRUST_REMOTE_CODE: bool = True
-
-# Multi-Token Prediction (MTP) configuration for DeepSeek V3
-# MTP enables speculative decoding using the model's built-in draft layer
-MTP_ENABLED: bool = True  # Feature flag to enable/disable MTP
-MTP_NUM_DRAFT_TOKENS: int = 1  # Number of tokens to draft (vLLM reports k=1 is optimal)

src/exo/worker/engines/mlx/generator/generate.py

@@ -19,13 +19,7 @@ from exo.shared.types.worker.runner_response import (
     GenerationResponse,
 )
 from exo.worker.engines.mlx import Model
-from exo.worker.engines.mlx.constants import (
-    KV_BITS,
-    KV_GROUP_SIZE,
-    MAX_TOKENS,
-    MTP_ENABLED,
-    MTP_NUM_DRAFT_TOKENS,
-)
+from exo.worker.engines.mlx.constants import KV_BITS, KV_GROUP_SIZE, MAX_TOKENS
 from exo.worker.engines.mlx.utils_mlx import (
     apply_chat_template,
     make_kv_cache,
@@ -121,11 +115,6 @@ def eos_ids_from_tokenizer(tokenizer: TokenizerWrapper) -> list[int]:
     return eos
 
 
-def _has_mtp_module(model: Model) -> bool:
-    """Check if the model has an attached MTP module."""
-    return hasattr(model, "mtp_module") and model.mtp_module is not None  # type: ignore[attr-defined]
-
-
 def mlx_generate(
     model: Model,
     tokenizer: TokenizerWrapper,
@@ -160,43 +149,6 @@ def mlx_generate(
     )
 
     max_tokens = task.max_tokens or MAX_TOKENS
-
-    # Check if we should use MTP speculative decoding
-    use_mtp = MTP_ENABLED and _has_mtp_module(model)
-
-    if use_mtp:
-        logger.info("Using MTP speculative decoding")
-        yield from _mlx_generate_with_mtp(
-            model=model,
-            tokenizer=tokenizer,
-            prompt=prompt,
-            max_tokens=max_tokens,
-            sampler=sampler,
-            logits_processors=logits_processors,
-            prompt_cache=caches,
-        )
-    else:
-        yield from _mlx_generate_standard(
-            model=model,
-            tokenizer=tokenizer,
-            prompt=prompt,
-            max_tokens=max_tokens,
-            sampler=sampler,
-            logits_processors=logits_processors,
-            prompt_cache=caches,
-        )
-
-
-def _mlx_generate_standard(
-    model: Model,
-    tokenizer: TokenizerWrapper,
-    prompt: str,
-    max_tokens: int,
-    sampler: Callable[[mx.array], mx.array],
-    logits_processors: list[Callable[[mx.array, mx.array], mx.array]],
-    prompt_cache: list[KVCache | Any],
-) -> Generator[GenerationResponse]:
-    """Standard generation path using mlx_lm stream_generate."""
     for out in stream_generate(
         model=model,
         tokenizer=tokenizer,
@@ -204,7 +156,7 @@ def _mlx_generate_standard(
         max_tokens=max_tokens,
         sampler=sampler,
         logits_processors=logits_processors,
-        prompt_cache=prompt_cache,
+        prompt_cache=caches,
         # TODO: Dynamically change prefill step size to be the maximum possible without timing out.
         prefill_step_size=2048,
         kv_group_size=KV_GROUP_SIZE,
@@ -239,64 +191,4 @@ def _mlx_generate_standard(
         if out.finish_reason is not None:
             break
 
-
-def _mlx_generate_with_mtp(
-    model: Model,
-    tokenizer: TokenizerWrapper,
-    prompt: str,
-    max_tokens: int,
-    sampler: Callable[[mx.array], mx.array],
-    logits_processors: list[Callable[[mx.array, mx.array], mx.array]],
-    prompt_cache: list[KVCache | Any],
-) -> Generator[GenerationResponse]:
-    """MTP speculative decoding generation path.
-
-    Uses the model's attached MTP module for speculative decoding,
-    which can provide 1.5-2x speedup with ~81% acceptance rate.
-    """
-    from exo.worker.engines.mlx.mtp.speculative_decode import mtp_speculative_generate
-
-    mtp_module = model.mtp_module  # type: ignore[attr-defined]
-
-    for out in mtp_speculative_generate(
-        model=model,
-        mtp_module=mtp_module,
-        tokenizer=tokenizer,
-        prompt=prompt,
-        max_tokens=max_tokens,
-        sampler=sampler,
-        logits_processors=logits_processors,
-        prompt_cache=prompt_cache,
-        num_draft_tokens=MTP_NUM_DRAFT_TOKENS,
-        prefill_step_size=2048,
-        kv_group_size=KV_GROUP_SIZE if KV_GROUP_SIZE is not None else 64,
-        kv_bits=KV_BITS,
-    ):
-        logger.info(f"{out.text} (from_draft={out.from_draft})")
-
-        stats: GenerationStats | None = None
-        if out.finish_reason is not None:
-            stats = GenerationStats(
-                prompt_tps=float(out.prompt_tps),
-                generation_tps=float(out.generation_tps),
-                prompt_tokens=int(out.prompt_tokens),
-                generation_tokens=int(out.generation_tokens),
-                peak_memory_usage=Memory.from_gb(out.peak_memory),
-            )
-
-            if out.finish_reason not in get_args(FinishReason):
-                logger.warning(
-                    f"Model generated unexpected finish_reason: {out.finish_reason}"
-                )
-
-        yield GenerationResponse(
-            text=out.text,
-            token=out.token,
-            finish_reason=cast(FinishReason | None, out.finish_reason),
-            stats=stats,
-        )
-
-        if out.finish_reason is not None:
-            break
-
         # TODO: Do we want an mx_barrier?

src/exo/worker/engines/mlx/mtp/__init__.py

@@ -1,6 +0,0 @@
-"""Multi-Token Prediction (MTP) module for DeepSeek V3 speculative decoding."""
-
-from exo.worker.engines.mlx.mtp.module import MTPModule
-from exo.worker.engines.mlx.mtp.speculative_decode import mtp_speculative_generate
-
-__all__ = ["MTPModule", "mtp_speculative_generate"]

src/exo/worker/engines/mlx/mtp/module.py

@@ -1,207 +0,0 @@
-"""MTP Module for DeepSeek V3 Multi-Token Prediction.
-
-The MTP architecture predicts one additional token ahead using:
-1. hnorm - RMSNorm for hidden state normalization
-2. enorm - RMSNorm for embedding normalization
-3. eh_proj - Linear(2*hidden_size -> hidden_size) projection
-4. transformer_block - Single decoder layer (attention + MLP)
-5. Shared embedding/lm_head from main model
-
-Forward pass:
-    h_norm = hnorm(hidden_state)
-    e_norm = enorm(embed(token))
-    projected = eh_proj(concat([h_norm, e_norm]))
-    new_hidden = transformer_block(projected)
-    logits = lm_head(output_norm(new_hidden))
-"""
-
-from typing import Any
-
-import mlx.core as mx
-import mlx.nn as nn
-from mlx_lm.models.cache import KVCache
-from mlx_lm.models.deepseek_v3 import (
-    DeepseekV3Attention,
-    DeepseekV3MLP,
-    ModelArgs,
-)
-
-MTP_LAYER_INDEX = 61
-
-
-class MTPModule(nn.Module):
-    """Multi-Token Prediction module for DeepSeek V3.
-
-    This module is initialized from the layer 61 weights that are normally
-    discarded during model loading. It enables speculative decoding by
-    predicting one token ahead using the hidden state from the main model.
-    """
-
-    def __init__(
-        self,
-        config: ModelArgs,
-        shared_embedding: nn.Embedding,
-        shared_lm_head: nn.Linear,
-        output_norm: nn.RMSNorm,
-    ) -> None:
-        super().__init__()
-        self.config = config
-
-        # MTP-specific normalization layers
-        self.hnorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.enorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-
-        # Projection: concatenated [hidden, embedding] -> hidden_size
-        self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False)
-
-        # Single transformer block for MTP
-        # Use a dense MLP since this is just a single layer
-        self.transformer_block = MTPTransformerBlock(config)
-
-        # Share embedding and lm_head with main model
-        self._shared_embedding = shared_embedding
-        self._shared_lm_head = shared_lm_head
-        self._output_norm = output_norm
-
-    def __call__(
-        self,
-        hidden_state: mx.array,
-        draft_token: mx.array,
-        cache: KVCache | None = None,
-        mask: mx.array | None = None,
-    ) -> tuple[mx.array, mx.array]:
-        """Forward pass for MTP.
-
-        Args:
-            hidden_state: Hidden state from main model [batch, seq_len, hidden_size]
-            draft_token: Token to embed and combine with hidden state [batch, seq_len]
-            cache: Optional KV cache for the MTP transformer block
-            mask: Optional attention mask
-
-        Returns:
-            tuple of (logits, new_hidden_state)
-        """
-        # Get embedding of draft token
-        embedding = self._shared_embedding(draft_token)
-
-        # Normalize hidden state and embedding
-        h_norm = self.hnorm(hidden_state)
-        e_norm = self.enorm(embedding)
-
-        # Project concatenated representation
-        concatenated = mx.concatenate([h_norm, e_norm], axis=-1)
-        projected = self.eh_proj(concatenated)
-
-        # Pass through single transformer block
-        new_hidden = self.transformer_block(projected, mask=mask, cache=cache)
-
-        # Apply output norm and get logits
-        normed_hidden = self._output_norm(new_hidden)
-        logits = self._shared_lm_head(normed_hidden)
-
-        return logits, new_hidden
-
-
-class MTPTransformerBlock(nn.Module):
-    """Single transformer block for MTP.
-
-    This is similar to DeepseekV3DecoderLayer but uses a dense MLP
-    instead of MoE since this is just for the single MTP layer.
-    """
-
-    def __init__(self, config: ModelArgs) -> None:
-        super().__init__()
-        self.self_attn = DeepseekV3Attention(config)
-        # MTP uses dense MLP, not MoE
-        self.mlp = DeepseekV3MLP(config)
-        self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.post_attention_layernorm = nn.RMSNorm(
-            config.hidden_size, eps=config.rms_norm_eps
-        )
-
-    def __call__(
-        self,
-        x: mx.array,
-        mask: mx.array | None = None,
-        cache: Any | None = None,
-    ) -> mx.array:
-        """Forward pass with residual connections."""
-        r = self.self_attn(self.input_layernorm(x), mask, cache)
-        h = x + r
-        r = self.mlp(self.post_attention_layernorm(h))
-        return h + r
-
-
-def extract_mtp_weights(weights: dict[str, mx.array]) -> dict[str, mx.array]:
-    """Extract MTP-specific weights from layer 61.
-
-    The MTP layer has these weight patterns:
-    - model.layers.61.enorm.weight -> MTP embedding normalization
-    - model.layers.61.hnorm.weight -> MTP hidden normalization
-    - model.layers.61.eh_proj.weight -> MTP projection layer
-    - model.layers.61.self_attn.* -> MTP attention
-    - model.layers.61.input_layernorm.* -> MTP layer norms
-    - model.layers.61.post_attention_layernorm.*
-    - model.layers.61.mlp.* -> MTP MLP (dense, not MoE)
-
-    Args:
-        weights: Full model weights dict
-
-    Returns:
-        Dict of MTP-specific weights with keys renamed for MTPModule
-    """
-    mtp_weights: dict[str, mx.array] = {}
-    mtp_prefix = f"model.layers.{MTP_LAYER_INDEX}."
-
-    for key, value in weights.items():
-        if key.startswith(mtp_prefix):
-            # Remove the layer prefix to get relative path
-            new_key = key[len(mtp_prefix) :]
-            mtp_weights[new_key] = value
-
-    return mtp_weights
-
-
-def load_mtp_weights_into_module(
-    mtp_module: MTPModule,
-    mtp_weights: dict[str, mx.array],
-) -> None:
-    """Load extracted MTP weights into the MTPModule.
-
-    Args:
-        mtp_module: The MTPModule instance to load weights into
-        mtp_weights: Extracted MTP weights from extract_mtp_weights()
-    """
-    # Map weight names to module attributes
-    weight_mapping: dict[str, str] = {
-        "enorm.weight": "enorm.weight",
-        "hnorm.weight": "hnorm.weight",
-        "eh_proj.weight": "eh_proj.weight",
-    }
-
-    # Load direct mappings
-    for src_name, dst_name in weight_mapping.items():
-        if src_name in mtp_weights:
-            parts = dst_name.split(".")
-            obj: Any = mtp_module
-            for part in parts[:-1]:
-                obj = getattr(obj, part)
-            setattr(obj, parts[-1], mtp_weights[src_name])
-
-    # Load transformer block weights (self_attn, mlp, layer norms)
-    transformer_prefixes = [
-        "self_attn",
-        "mlp",
-        "input_layernorm",
-        "post_attention_layernorm",
-    ]
-
-    for prefix in transformer_prefixes:
-        for key, value in mtp_weights.items():
-            if key.startswith(prefix):
-                # Navigate to the correct attribute
-                parts = key.split(".")
-                obj = mtp_module.transformer_block
-                for part in parts[:-1]:
-                    obj = getattr(obj, part)
-                setattr(obj, parts[-1], value)

src/exo/worker/engines/mlx/mtp/speculative_decode.py

@@ -1,506 +0,0 @@
-"""MTP Speculative Decoding for DeepSeek V3.
-
-This module implements speculative decoding using the Multi-Token Prediction (MTP)
-layer from DeepSeek V3. The key difference from standard speculative decoding is
-that MTP requires hidden states from the main model, not just token predictions.
-
-Based on vLLM/SGLang research:
-- 81-82% acceptance rate with k=1
-- 1.5-2x speedup at low QPS
-"""
-
-import functools
-import time
-from collections.abc import Callable, Generator
-from dataclasses import dataclass
-from typing import Any, cast
-
-import mlx.core as mx
-import mlx.nn as nn
-from mlx_lm.models import cache
-from mlx_lm.models.cache import KVCache
-from mlx_lm.tokenizer_utils import TokenizerWrapper
-
-from exo.worker.engines.mlx.mtp.module import MTPModule
-
-# Generation stream for async operations
-generation_stream = mx.new_stream(mx.default_device())
-
-
-@dataclass
-class MTPGenerationResponse:
-    """Response from MTP speculative generation.
-
-    Attributes:
-        text: The next segment of decoded text.
-        token: The next token.
-        logprobs: A vector of log probabilities.
-        from_draft: Whether the token was generated by the MTP draft module.
-        prompt_tokens: The number of tokens in the prompt.
-        prompt_tps: The prompt processing tokens-per-second.
-        generation_tokens: The number of generated tokens.
-        generation_tps: The tokens-per-second for generation.
-        peak_memory: The peak memory used so far in GB.
-        finish_reason: The reason the response is being sent: "length", "stop" or None.
-    """
-
-    text: str
-    token: int
-    logprobs: mx.array
-    from_draft: bool
-    prompt_tokens: int
-    prompt_tps: float
-    generation_tokens: int
-    generation_tps: float
-    peak_memory: float
-    finish_reason: str | None = None
-
-
-def maybe_quantize_kv_cache(
-    prompt_cache: list[Any],
-    quantized_kv_start: int,
-    kv_group_size: int,
-    kv_bits: int | None,
-) -> None:
-    """Quantize KV cache entries if needed."""
-    if kv_bits is None:
-        return
-    for e, c in enumerate(prompt_cache):
-        if (
-            hasattr(c, "to_quantized")
-            and hasattr(c, "offset")
-            and c.offset >= quantized_kv_start
-        ):
-            prompt_cache[e] = c.to_quantized(group_size=kv_group_size, bits=kv_bits)
-
-
-class ModelWithHiddenStates(nn.Module):
-    """Wrapper to extract hidden states before lm_head.
-
-    This wrapper allows capturing the hidden states from the transformer
-    layers before the final lm_head projection, which is needed for MTP.
-    """
-
-    def __init__(self, base_model: nn.Module) -> None:
-        super().__init__()
-        self._base = base_model
-
-    def forward_with_hidden(
-        self,
-        inputs: mx.array,
-        model_cache: list[Any] | None = None,
-    ) -> tuple[mx.array, mx.array]:
-        """Forward pass that returns both logits and hidden states.
-
-        Args:
-            inputs: Input token ids
-            model_cache: KV cache
-
-        Returns:
-            Tuple of (logits, hidden_states)
-        """
-        # Call the inner model (transformer layers + norm)
-        hidden: mx.array = self._base.model(inputs, model_cache)
-        # Get logits from lm_head
-        logits: mx.array = self._base.lm_head(hidden)
-        return logits, hidden
-
-    def forward(
-        self,
-        inputs: mx.array,
-        model_cache: list[Any] | None = None,
-    ) -> mx.array:
-        """Standard forward pass returning only logits."""
-        return cast(mx.array, self._base(inputs, cache=model_cache))
-
-    @property
-    def layers(self) -> list[nn.Module]:
-        """Access layers for cache creation."""
-        return cast(list[nn.Module], self._base.layers)
-
-
-def mtp_speculative_generate_step(
-    prompt: mx.array,
-    model: nn.Module,
-    mtp_module: MTPModule,
-    *,
-    num_draft_tokens: int = 1,
-    max_tokens: int = 256,
-    sampler: Callable[[mx.array], mx.array] | None = None,
-    logits_processors: list[Callable[[mx.array, mx.array], mx.array]] | None = None,
-    prompt_cache: list[Any] | None = None,
-    mtp_cache: KVCache | None = None,
-    prefill_step_size: int = 512,
-    kv_bits: int | None = None,
-    kv_group_size: int = 64,
-    quantized_kv_start: int = 0,
-) -> Generator[tuple[int, mx.array, bool], None, None]:
-    """MTP speculative decoding generator.
-
-    Unlike standard speculative decoding where the draft model only needs tokens,
-    MTP requires the hidden states from the main model. This generator:
-
-    1. Runs the main model to get logits AND hidden states
-    2. Uses MTP module with hidden state + sampled token to predict next token
-    3. Verifies MTP predictions with the main model
-    4. Accepts/rejects based on matching
-
-    Args:
-        prompt: The input prompt as token ids
-        model: The main model (must support return_hidden=True)
-        mtp_module: The MTP module for draft prediction
-        num_draft_tokens: Number of tokens to draft (typically 1 for MTP)
-        max_tokens: Maximum number of tokens to generate
-        sampler: Optional sampler function for token selection
-        logits_processors: Optional list of logits processors
-        prompt_cache: KV cache for the main model
-        mtp_cache: KV cache for the MTP module
-        prefill_step_size: Step size for prompt processing
-        kv_bits: Bits for KV cache quantization
-        kv_group_size: Group size for KV cache quantization
-        quantized_kv_start: Step to begin cache quantization
-
-    Yields:
-        Tuple of (token, logprobs, from_draft)
-    """
-    y = prompt.astype(mx.uint32)
-    prev_tokens: mx.array | None = None
-
-    # Wrap model to get hidden states
-    wrapped_model = (
-        model
-        if isinstance(model, ModelWithHiddenStates)
-        else ModelWithHiddenStates(model)
-    )
-
-    # Create caches if needed
-    if prompt_cache is None:
-        prompt_cache = cache.make_prompt_cache(model)
-    if mtp_cache is None:
-        mtp_cache = KVCache()
-
-    final_sampler = (
-        sampler if sampler is not None else (lambda x: mx.argmax(x, axis=-1))
-    )
-
-    quantize_cache_fn = functools.partial(
-        maybe_quantize_kv_cache,
-        quantized_kv_start=quantized_kv_start,
-        kv_group_size=kv_group_size,
-        kv_bits=kv_bits,
-    )
-
-    def _process_and_sample(
-        tokens: mx.array | None,
-        logits: mx.array,
-    ) -> tuple[mx.array, mx.array]:
-        """Process logits and sample tokens."""
-        nonlocal logits_processors
-        processed_logits = logits
-        if logits_processors:
-            for processor in logits_processors:
-                processed_logits = processor(
-                    tokens if tokens is not None else mx.array([]), processed_logits
-                )
-
-        logprobs = processed_logits - mx.logsumexp(
-            processed_logits, axis=-1, keepdims=True
-        )
-        sampled = final_sampler(logprobs)
-        return sampled, logprobs
-
-    def _main_model_step_with_hidden(
-        input_y: mx.array,
-    ) -> tuple[mx.array, mx.array, mx.array]:
-        """Run main model step with hidden state return."""
-        nonlocal prev_tokens
-
-        with mx.stream(generation_stream):
-            logits, hidden = wrapped_model.forward_with_hidden(
-                input_y[None], prompt_cache
-            )
-            logits = logits[:, -1, :]
-            quantize_cache_fn(prompt_cache)
-
-            if logits_processors:
-                prev_tokens = (
-                    mx.concatenate([prev_tokens, input_y])
-                    if prev_tokens is not None
-                    else input_y
-                )
-
-            sampled, logprobs_result = _process_and_sample(prev_tokens, logits)
-            return sampled, logprobs_result.squeeze(0), hidden[:, -1:, :]
-
-    def _main_model_step(
-        input_y: mx.array,
-    ) -> tuple[mx.array, mx.array]:
-        """Run main model step without hidden state."""
-        nonlocal prev_tokens
-
-        with mx.stream(generation_stream):
-            logits = wrapped_model.forward(input_y[None], prompt_cache)
-            logits = logits[:, -1, :]
-            quantize_cache_fn(prompt_cache)
-
-            if logits_processors:
-                prev_tokens = (
-                    mx.concatenate([prev_tokens, input_y])
-                    if prev_tokens is not None
-                    else input_y
-                )
-
-            sampled, logprobs_result = _process_and_sample(prev_tokens, logits)
-            return sampled, logprobs_result.squeeze(0)
-
-    def _mtp_draft(
-        hidden_state: mx.array,
-        draft_token: mx.array,
-    ) -> tuple[mx.array, mx.array]:
-        """Generate draft token using MTP module."""
-        with mx.stream(generation_stream):
-            logits, new_hidden = mtp_module(
-                hidden_state,
-                draft_token,
-                cache=mtp_cache,
-            )
-            logits = logits[:, -1, :]
-            sampled, _ = _process_and_sample(None, logits)
-            return sampled, new_hidden
-
-    def _prefill(input_y: mx.array) -> mx.array:
-        """Prefill the prompt cache."""
-        result_y = input_y
-        while result_y.size > prefill_step_size:
-            _ = wrapped_model.forward(result_y[:prefill_step_size][None], prompt_cache)
-            quantize_cache_fn(prompt_cache)
-            mx.eval([c.state for c in prompt_cache])
-            result_y = result_y[prefill_step_size:]
-            mx.clear_cache()
-        return result_y
-
-    def _rewind_cache(num_draft: int, num_accept: int) -> None:
-        """Rewind caches after rejection."""
-        cache.trim_prompt_cache(prompt_cache, num_draft - num_accept)
-
-    # Prefill phase
-    with mx.stream(generation_stream):
-        y = _prefill(y)
-
-    ntoks = 0
-    num_draft = 0
-    n_accepted = 0
-    last_hidden: mx.array | None = None
-
-    try:
-        # Initial step to get first token and hidden state
-        sampled, logprobs, last_hidden = _main_model_step_with_hidden(y)
-        mx.eval(sampled, logprobs, last_hidden)
-
-        y = sampled
-        current_logprobs = logprobs
-
-        while ntoks < max_tokens:
-            # Draft phase: use MTP to predict next token
-            num_draft = min(max_tokens - ntoks - 1, num_draft_tokens)
-
-            if num_draft > 0 and last_hidden is not None:
-                # Use MTP to draft
-                draft_token, draft_hidden = _mtp_draft(last_hidden, y)
-                mx.eval(draft_token, draft_hidden)
-
-                # Verify with main model
-                # Feed the drafted token to main model
-                verify_input = mx.concatenate([y, draft_token.flatten()])
-                verify_sampled, verify_logprobs, new_hidden = (
-                    _main_model_step_with_hidden(verify_input)
-                )
-                mx.eval(verify_sampled, verify_logprobs, new_hidden)
-
-                # Check if draft matches verification
-                draft_token_val = int(draft_token.item())
-                verify_token_val = (
-                    int(verify_sampled[0].item())
-                    if verify_sampled.shape[0] > 1
-                    else int(verify_sampled.item())
-                )
-
-                # Yield the current token (not from draft)
-                ntoks += 1
-                yield int(y.item()), current_logprobs, False
-
-                if ntoks >= max_tokens:
-                    break
-
-                if draft_token_val == verify_token_val:
-                    # Draft accepted
-                    n_accepted += 1
-                    ntoks += 1
-                    draft_logprobs = (
-                        verify_logprobs[0]
-                        if verify_logprobs.ndim > 1
-                        else verify_logprobs
-                    )
-                    yield draft_token_val, draft_logprobs, True
-
-                    if ntoks >= max_tokens:
-                        break
-
-                    # Continue with the token after the draft
-                    y = (
-                        verify_sampled[-1:]
-                        if verify_sampled.ndim > 0 and verify_sampled.shape[0] > 1
-                        else verify_sampled
-                    )
-                    current_logprobs = (
-                        verify_logprobs[-1]
-                        if verify_logprobs.ndim > 1
-                        else verify_logprobs
-                    )
-                    last_hidden = new_hidden
-                else:
-                    # Draft rejected - rewind and use verified token
-                    _rewind_cache(1, 0)
-                    y = (
-                        verify_sampled[:1]
-                        if verify_sampled.ndim > 0 and verify_sampled.shape[0] > 1
-                        else verify_sampled
-                    )
-                    current_logprobs = (
-                        verify_logprobs[0]
-                        if verify_logprobs.ndim > 1
-                        else verify_logprobs
-                    )
-                    last_hidden = (
-                        new_hidden[:, :1, :] if new_hidden is not None else None
-                    )
-            else:
-                # No drafting, just do normal generation
-                ntoks += 1
-                yield int(y.item()), current_logprobs, False
-
-                if ntoks >= max_tokens:
-                    break
-
-                sampled, logprobs, last_hidden = _main_model_step_with_hidden(y)
-                mx.eval(sampled, logprobs, last_hidden)
-
-                y = sampled
-                current_logprobs = logprobs
-
-            if ntoks % 256 == 0:
-                mx.clear_cache()
-
-    finally:
-        _rewind_cache(num_draft, n_accepted)
-
-
-def mtp_speculative_generate(
-    model: nn.Module,
-    mtp_module: MTPModule,
-    tokenizer: TokenizerWrapper,
-    prompt: str | mx.array | list[int],
-    max_tokens: int = 256,
-    sampler: Callable[[mx.array], mx.array] | None = None,
-    logits_processors: list[Callable[[mx.array, mx.array], mx.array]] | None = None,
-    prompt_cache: list[Any] | None = None,
-    num_draft_tokens: int = 1,
-    prefill_step_size: int = 512,
-    kv_group_size: int = 64,
-    kv_bits: int | None = None,
-) -> Generator[MTPGenerationResponse, None, None]:
-    """High-level MTP speculative generation with text output.
-
-    Args:
-        model: The main model
-        mtp_module: The MTP module for draft prediction
-        tokenizer: Tokenizer for encoding/decoding
-        prompt: Input prompt (string, array, or token list)
-        max_tokens: Maximum tokens to generate
-        sampler: Optional sampler function
-        logits_processors: Optional logits processors
-        prompt_cache: Optional KV cache
-        num_draft_tokens: Number of draft tokens
-        prefill_step_size: Prefill step size
-        kv_group_size: KV group size
-        kv_bits: KV bits
-
-    Yields:
-        MTPGenerationResponse objects with text and metadata
-    """
-    if not isinstance(prompt, mx.array):
-        if isinstance(prompt, str):
-            bos_token = getattr(tokenizer, "bos_token", None)
-            add_special_tokens = bos_token is None or not prompt.startswith(
-                str(bos_token)
-            )
-            encoded: list[int] = tokenizer.encode(
-                prompt, add_special_tokens=add_special_tokens
-            )
-            prompt = mx.array(encoded)
-        else:
-            prompt = mx.array(prompt)
-
-    detokenizer = tokenizer.detokenizer
-    eos_token_ids: list[int] = getattr(tokenizer, "eos_token_ids", [])
-
-    token_generator = mtp_speculative_generate_step(
-        prompt,
-        model,
-        mtp_module,
-        max_tokens=max_tokens,
-        sampler=sampler,
-        logits_processors=logits_processors,
-        prompt_cache=prompt_cache,
-        num_draft_tokens=num_draft_tokens,
-        prefill_step_size=prefill_step_size,
-        kv_group_size=kv_group_size,
-        kv_bits=kv_bits,
-    )
-
-    tic = time.perf_counter()
-    prompt_tps = 0.0
-    token = 0
-    logprobs: mx.array = mx.array([0.0])
-    from_draft = False
-    n = 0
-
-    for n, (token, logprobs, from_draft) in enumerate(token_generator):
-        if n == 0:
-            prompt_time = time.perf_counter() - tic
-            prompt_tps = float(prompt.size) / prompt_time
-            tic = time.perf_counter()
-
-        if token in eos_token_ids:
-            break
-
-        detokenizer.add_token(token)
-        if (n + 1) == max_tokens:
-            break
-
-        yield MTPGenerationResponse(
-            text=str(detokenizer.last_segment),
-            token=token,
-            logprobs=logprobs,
-            from_draft=from_draft,
-            prompt_tokens=int(prompt.size),
-            prompt_tps=prompt_tps,
-            generation_tokens=n + 1,
-            generation_tps=(n + 1) / (time.perf_counter() - tic),
-            peak_memory=mx.get_peak_memory() / 1e9,
-            finish_reason=None,
-        )
-
-    detokenizer.finalize()
-    yield MTPGenerationResponse(
-        text=str(detokenizer.last_segment),
-        token=token,
-        logprobs=logprobs,
-        from_draft=from_draft,
-        prompt_tokens=int(prompt.size),
-        prompt_tps=prompt_tps,
-        generation_tokens=n + 1,
-        generation_tps=(n + 1) / (time.perf_counter() - tic),
-        peak_memory=mx.get_peak_memory() / 1e9,
-        finish_reason="stop" if token in eos_token_ids else "length",
-    )

src/exo/worker/engines/mlx/mtp/tests/__init__.py

@@ -1 +0,0 @@
-"""Tests for MTP module."""

src/exo/worker/engines/mlx/mtp/tests/test_mtp_module.py

@@ -1,412 +0,0 @@
-"""Unit tests for MTP module components."""
-
-import mlx.core as mx
-import mlx.nn as nn
-import pytest
-
-from exo.worker.engines.mlx.mtp.module import (
-    MTP_LAYER_INDEX,
-    MTPModule,
-    MTPTransformerBlock,
-    extract_mtp_weights,
-    load_mtp_weights_into_module,
-)
-
-
-class MockModelArgs:
-    """Mock ModelArgs for testing without importing deepseek_v3."""
-
-    def __init__(
-        self,
-        hidden_size: int = 256,
-        intermediate_size: int = 512,
-        num_attention_heads: int = 4,
-        num_key_value_heads: int = 4,
-        rms_norm_eps: float = 1e-6,
-        vocab_size: int = 1000,
-        q_lora_rank: int | None = None,
-        kv_lora_rank: int = 64,
-        qk_rope_head_dim: int = 16,
-        v_head_dim: int = 32,
-        qk_nope_head_dim: int = 32,
-        rope_theta: float = 10000.0,
-        rope_scaling: dict | None = None,
-        attention_bias: bool = False,
-        max_position_embeddings: int = 2048,
-    ):
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_attention_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        self.rms_norm_eps = rms_norm_eps
-        self.vocab_size = vocab_size
-        self.q_lora_rank = q_lora_rank
-        self.kv_lora_rank = kv_lora_rank
-        self.qk_rope_head_dim = qk_rope_head_dim
-        self.v_head_dim = v_head_dim
-        self.qk_nope_head_dim = qk_nope_head_dim
-        self.rope_theta = rope_theta
-        self.rope_scaling = rope_scaling
-        self.attention_bias = attention_bias
-        self.max_position_embeddings = max_position_embeddings
-
-
-class TestExtractMTPWeights:
-    """Tests for extract_mtp_weights function."""
-
-    def test_extracts_layer_61_weights(self) -> None:
-        """Should extract only layer 61 weights."""
-        weights = {
-            "model.layers.60.self_attn.weight": mx.zeros((10, 10)),
-            "model.layers.61.enorm.weight": mx.ones((10,)),
-            "model.layers.61.hnorm.weight": mx.ones((10,)) * 2,
-            "model.layers.61.eh_proj.weight": mx.ones((10, 20)),
-            "model.layers.62.self_attn.weight": mx.zeros((10, 10)),
-            "model.embed_tokens.weight": mx.zeros((100, 10)),
-        }
-
-        mtp_weights = extract_mtp_weights(weights)
-
-        assert len(mtp_weights) == 3
-        assert "enorm.weight" in mtp_weights
-        assert "hnorm.weight" in mtp_weights
-        assert "eh_proj.weight" in mtp_weights
-        # Check values are preserved
-        assert mx.allclose(mtp_weights["enorm.weight"], mx.ones((10,)))
-        assert mx.allclose(mtp_weights["hnorm.weight"], mx.ones((10,)) * 2)
-
-    def test_returns_empty_dict_when_no_layer_61(self) -> None:
-        """Should return empty dict when layer 61 doesn't exist."""
-        weights = {
-            "model.layers.0.self_attn.weight": mx.zeros((10, 10)),
-            "model.layers.60.self_attn.weight": mx.zeros((10, 10)),
-        }
-
-        mtp_weights = extract_mtp_weights(weights)
-
-        assert len(mtp_weights) == 0
-
-    def test_handles_nested_layer_61_weights(self) -> None:
-        """Should handle nested weight paths like self_attn.q_proj.weight."""
-        weights = {
-            f"model.layers.{MTP_LAYER_INDEX}.self_attn.q_a_proj.weight": mx.zeros(
-                (10, 10)
-            ),
-            f"model.layers.{MTP_LAYER_INDEX}.mlp.gate_proj.weight": mx.zeros((20, 10)),
-        }
-
-        mtp_weights = extract_mtp_weights(weights)
-
-        assert "self_attn.q_a_proj.weight" in mtp_weights
-        assert "mlp.gate_proj.weight" in mtp_weights
-
-
-class TestMTPTransformerBlock:
-    """Tests for MTPTransformerBlock."""
-
-    @pytest.fixture
-    def config(self) -> MockModelArgs:
-        return MockModelArgs(
-            hidden_size=64, intermediate_size=128, num_attention_heads=2
-        )
-
-    def test_forward_shape(self, config: MockModelArgs) -> None:
-        """Forward pass should preserve input shape."""
-        # Skip if deepseek_v3 imports fail (CI without mlx_lm)
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        block = MTPTransformerBlock(config)  # type: ignore[arg-type]
-        x = mx.random.normal((1, 5, config.hidden_size))
-
-        output = block(x)
-
-        assert output.shape == x.shape
-
-    def test_forward_with_mask(self, config: MockModelArgs) -> None:
-        """Forward pass should work with attention mask."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        block = MTPTransformerBlock(config)  # type: ignore[arg-type]
-        x = mx.random.normal((1, 5, config.hidden_size))
-        # Create causal mask
-        mask = mx.triu(mx.full((5, 5), float("-inf")), k=1)
-
-        output = block(x, mask=mask)
-
-        assert output.shape == x.shape
-
-
-class TestMTPModule:
-    """Tests for MTPModule."""
-
-    @pytest.fixture
-    def config(self) -> MockModelArgs:
-        return MockModelArgs(
-            hidden_size=64,
-            intermediate_size=128,
-            num_attention_heads=2,
-            vocab_size=100,
-        )
-
-    @pytest.fixture
-    def shared_components(
-        self, config: MockModelArgs
-    ) -> tuple[nn.Embedding, nn.Linear, nn.RMSNorm]:
-        embedding = nn.Embedding(config.vocab_size, config.hidden_size)
-        lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-        output_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        return embedding, lm_head, output_norm
-
-    def test_initialization(
-        self,
-        config: MockModelArgs,
-        shared_components: tuple[nn.Embedding, nn.Linear, nn.RMSNorm],
-    ) -> None:
-        """MTPModule should initialize with correct components."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        embedding, lm_head, output_norm = shared_components
-        mtp = MTPModule(
-            config=config,  # type: ignore[arg-type]
-            shared_embedding=embedding,
-            shared_lm_head=lm_head,
-            output_norm=output_norm,
-        )
-
-        assert mtp.hnorm is not None
-        assert mtp.enorm is not None
-        assert mtp.eh_proj is not None
-        assert mtp.transformer_block is not None
-
-    def test_forward_output_shapes(
-        self,
-        config: MockModelArgs,
-        shared_components: tuple[nn.Embedding, nn.Linear, nn.RMSNorm],
-    ) -> None:
-        """Forward pass should return correct output shapes."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        embedding, lm_head, output_norm = shared_components
-        mtp = MTPModule(
-            config=config,  # type: ignore[arg-type]
-            shared_embedding=embedding,
-            shared_lm_head=lm_head,
-            output_norm=output_norm,
-        )
-
-        batch_size = 2
-        seq_len = 1
-        hidden_state = mx.random.normal((batch_size, seq_len, config.hidden_size))
-        draft_token = mx.array([[5], [10]])  # [batch, seq_len]
-
-        logits, new_hidden = mtp(hidden_state, draft_token)
-
-        assert logits.shape == (batch_size, seq_len, config.vocab_size)
-        assert new_hidden.shape == (batch_size, seq_len, config.hidden_size)
-
-    def test_shares_embedding_and_lm_head(
-        self,
-        config: MockModelArgs,
-        shared_components: tuple[nn.Embedding, nn.Linear, nn.RMSNorm],
-    ) -> None:
-        """MTPModule should use shared embedding and lm_head."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        embedding, lm_head, output_norm = shared_components
-        mtp = MTPModule(
-            config=config,  # type: ignore[arg-type]
-            shared_embedding=embedding,
-            shared_lm_head=lm_head,
-            output_norm=output_norm,
-        )
-
-        # Verify they're the same objects
-        assert mtp._shared_embedding is embedding
-        assert mtp._shared_lm_head is lm_head
-        assert mtp._output_norm is output_norm
-
-
-class TestLoadMTPWeights:
-    """Tests for load_mtp_weights_into_module."""
-
-    @pytest.fixture
-    def config(self) -> MockModelArgs:
-        return MockModelArgs(
-            hidden_size=64,
-            intermediate_size=128,
-            num_attention_heads=2,
-            vocab_size=100,
-        )
-
-    def test_loads_norm_weights(self, config: MockModelArgs) -> None:
-        """Should load enorm and hnorm weights."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        embedding = nn.Embedding(config.vocab_size, config.hidden_size)
-        lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-        output_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-
-        mtp = MTPModule(
-            config=config,  # type: ignore[arg-type]
-            shared_embedding=embedding,
-            shared_lm_head=lm_head,
-            output_norm=output_norm,
-        )
-
-        # Create test weights
-        test_enorm = mx.ones((config.hidden_size,)) * 3.0
-        test_hnorm = mx.ones((config.hidden_size,)) * 5.0
-        mtp_weights = {
-            "enorm.weight": test_enorm,
-            "hnorm.weight": test_hnorm,
-        }
-
-        load_mtp_weights_into_module(mtp, mtp_weights)
-
-        assert mx.allclose(mtp.enorm.weight, test_enorm)
-        assert mx.allclose(mtp.hnorm.weight, test_hnorm)
-
-
-class TestSanitizePatch:
-    """Tests for the sanitize patching logic."""
-
-    def test_patch_preserves_layer_61(self) -> None:
-        """Patching sanitize should preserve layer 61 weights."""
-        from exo.worker.engines.mlx.utils_mlx import (
-            _patch_deepseek_sanitize_for_mtp,
-            _restore_deepseek_sanitize,
-        )
-
-        deepseek_v3 = pytest.importorskip("mlx_lm.models.deepseek_v3")
-        model_cls = deepseek_v3.Model
-
-        # Get original sanitize behavior
-        original_sanitize = model_cls.sanitize
-
-        try:
-            # Apply patch
-            _patch_deepseek_sanitize_for_mtp()
-
-            # Note: we can't easily test the full sanitize without a real model
-            # This test verifies the patch is applied
-            assert model_cls.sanitize is not original_sanitize
-
-        finally:
-            _restore_deepseek_sanitize()
-            # Verify restore worked
-            assert model_cls.sanitize is original_sanitize
-
-    def test_restore_sanitize(self) -> None:
-        """Restoring sanitize should return to original behavior."""
-        from exo.worker.engines.mlx.utils_mlx import (
-            _patch_deepseek_sanitize_for_mtp,
-            _restore_deepseek_sanitize,
-        )
-
-        deepseek_v3 = pytest.importorskip("mlx_lm.models.deepseek_v3")
-        model_cls = deepseek_v3.Model
-
-        original_sanitize = model_cls.sanitize
-
-        _patch_deepseek_sanitize_for_mtp()
-        assert model_cls.sanitize is not original_sanitize
-
-        _restore_deepseek_sanitize()
-        assert model_cls.sanitize is original_sanitize
-
-    def test_double_patch_is_safe(self) -> None:
-        """Calling patch twice should be safe (idempotent)."""
-        from exo.worker.engines.mlx.utils_mlx import (
-            _patch_deepseek_sanitize_for_mtp,
-            _restore_deepseek_sanitize,
-        )
-
-        deepseek_v3 = pytest.importorskip("mlx_lm.models.deepseek_v3")
-        model_cls = deepseek_v3.Model
-
-        original_sanitize = model_cls.sanitize
-
-        try:
-            _patch_deepseek_sanitize_for_mtp()
-            patched_sanitize = model_cls.sanitize
-
-            # Patch again - should be no-op
-            _patch_deepseek_sanitize_for_mtp()
-            assert model_cls.sanitize is patched_sanitize
-
-        finally:
-            _restore_deepseek_sanitize()
-            assert model_cls.sanitize is original_sanitize
-
-
-class TestModelIdDetection:
-    """Tests for DeepSeek V3 model ID detection."""
-
-    def test_detects_deepseek_v3(self) -> None:
-        """Should detect DeepSeek V3 model IDs."""
-        from exo.worker.engines.mlx.utils_mlx import _might_be_deepseek_v3
-
-        assert _might_be_deepseek_v3("deepseek-ai/DeepSeek-V3")
-        assert _might_be_deepseek_v3("deepseek-ai/deepseek-v3-base")
-        assert _might_be_deepseek_v3("mlx-community/DeepSeek-V3-4bit")
-
-    def test_detects_deepseek_r1(self) -> None:
-        """Should detect DeepSeek R1 model IDs (also uses MTP)."""
-        from exo.worker.engines.mlx.utils_mlx import _might_be_deepseek_v3
-
-        assert _might_be_deepseek_v3("deepseek-ai/DeepSeek-R1")
-        assert _might_be_deepseek_v3("mlx-community/DeepSeek-R1-4bit")
-
-    def test_rejects_non_deepseek(self) -> None:
-        """Should reject non-DeepSeek model IDs."""
-        from exo.worker.engines.mlx.utils_mlx import _might_be_deepseek_v3
-
-        assert not _might_be_deepseek_v3("meta-llama/Llama-3-70B")
-        assert not _might_be_deepseek_v3("mistralai/Mixtral-8x7B")
-        assert not _might_be_deepseek_v3("deepseek-ai/DeepSeek-V2")  # V2, not V3
-
-    def test_case_insensitive(self) -> None:
-        """Detection should be case insensitive."""
-        from exo.worker.engines.mlx.utils_mlx import _might_be_deepseek_v3
-
-        assert _might_be_deepseek_v3("DEEPSEEK-AI/DEEPSEEK-V3")
-        assert _might_be_deepseek_v3("DeepSeek-AI/deepseek-v3")
-
-
-class TestFlattenParams:
-    """Tests for parameter flattening utility."""
-
-    def test_flattens_nested_dict(self) -> None:
-        """Should flatten nested parameter dict."""
-        from exo.worker.engines.mlx.utils_mlx import _flatten_params
-
-        params = {
-            "model": {
-                "layers": {
-                    "0": {
-                        "weight": mx.zeros((10,)),
-                    }
-                },
-                "embed": mx.ones((5,)),
-            }
-        }
-
-        flat = _flatten_params(params)
-
-        assert "model.layers.0.weight" in flat
-        assert "model.embed" in flat
-        assert mx.allclose(flat["model.layers.0.weight"], mx.zeros((10,)))
-        assert mx.allclose(flat["model.embed"], mx.ones((5,)))
-
-    def test_handles_flat_dict(self) -> None:
-        """Should handle already-flat dict."""
-        from exo.worker.engines.mlx.utils_mlx import _flatten_params
-
-        params = {
-            "weight": mx.zeros((10,)),
-            "bias": mx.ones((10,)),
-        }
-
-        flat = _flatten_params(params)
-
-        assert flat == params

src/exo/worker/engines/mlx/mtp/tests/test_speculative_decode.py

@@ -1,253 +0,0 @@
-"""Unit tests for MTP speculative decoding."""
-
-import mlx.core as mx
-import mlx.nn as nn
-import pytest
-
-from exo.worker.engines.mlx.mtp.speculative_decode import (
-    ModelWithHiddenStates,
-    maybe_quantize_kv_cache,
-)
-
-
-class MockModel(nn.Module):
-    """Mock model for testing speculative decoding."""
-
-    def __init__(self, hidden_size: int = 64, vocab_size: int = 100) -> None:
-        super().__init__()
-        self.hidden_size = hidden_size
-        self.vocab_size = vocab_size
-
-        # Create simple model components
-        self.model = MockInnerModel(hidden_size)
-        self.lm_head = nn.Linear(hidden_size, vocab_size, bias=False)
-        self._layers = [nn.Linear(hidden_size, hidden_size) for _ in range(3)]
-
-    def __call__(
-        self,
-        inputs: mx.array,
-        cache: list | None = None,
-    ) -> mx.array:
-        hidden = self.model(inputs, cache)
-        return self.lm_head(hidden)
-
-    @property
-    def layers(self) -> list[nn.Module]:
-        return self._layers
-
-
-class MockInnerModel(nn.Module):
-    """Mock inner model (like DeepseekV3Model)."""
-
-    def __init__(self, hidden_size: int) -> None:
-        super().__init__()
-        self.embed_tokens = nn.Embedding(100, hidden_size)
-        self.norm = nn.RMSNorm(hidden_size)
-
-    def __call__(
-        self,
-        inputs: mx.array,
-        cache: list | None = None,
-    ) -> mx.array:
-        # Simple embedding + norm
-        embedded = self.embed_tokens(inputs)
-        return self.norm(embedded)
-
-
-class TestModelWithHiddenStates:
-    """Tests for ModelWithHiddenStates wrapper."""
-
-    @pytest.fixture
-    def mock_model(self) -> MockModel:
-        return MockModel(hidden_size=64, vocab_size=100)
-
-    def test_forward_returns_logits(self, mock_model: MockModel) -> None:
-        """Standard forward should return logits."""
-        wrapped = ModelWithHiddenStates(mock_model)
-        inputs = mx.array([[1, 2, 3]])
-
-        logits = wrapped.forward(inputs)
-
-        assert logits.shape == (1, 3, mock_model.vocab_size)
-
-    def test_forward_with_hidden_returns_tuple(self, mock_model: MockModel) -> None:
-        """Forward with hidden should return (logits, hidden)."""
-        wrapped = ModelWithHiddenStates(mock_model)
-        inputs = mx.array([[1, 2, 3]])
-
-        logits, hidden = wrapped.forward_with_hidden(inputs)
-
-        assert logits.shape == (1, 3, mock_model.vocab_size)
-        assert hidden.shape == (1, 3, mock_model.hidden_size)
-
-    def test_layers_property(self, mock_model: MockModel) -> None:
-        """Should expose layers property from base model."""
-        wrapped = ModelWithHiddenStates(mock_model)
-
-        assert wrapped.layers == mock_model.layers
-        assert len(wrapped.layers) == 3
-
-
-class TestMaybeQuantizeKVCache:
-    """Tests for KV cache quantization."""
-
-    def test_no_quantization_when_bits_none(self) -> None:
-        """Should not quantize when kv_bits is None."""
-        cache = [MockCache(offset=100)]
-
-        maybe_quantize_kv_cache(
-            cache,
-            quantized_kv_start=50,
-            kv_group_size=64,
-            kv_bits=None,
-        )
-
-        # Cache should be unchanged
-        assert not hasattr(cache[0], "quantized")
-
-    def test_respects_quantized_kv_start(self) -> None:
-        """Should only quantize caches past the start threshold."""
-        cache_below = MockCache(offset=30)
-        cache_above = MockCache(offset=100)
-        caches = [cache_below, cache_above]
-
-        maybe_quantize_kv_cache(
-            caches,
-            quantized_kv_start=50,
-            kv_group_size=64,
-            kv_bits=4,
-        )
-
-        # Only cache_above should be quantized
-        assert not getattr(cache_below, "was_quantized", False)
-        assert getattr(caches[1], "was_quantized", False)
-
-
-class MockCache:
-    """Mock KV cache for testing."""
-
-    def __init__(self, offset: int = 0) -> None:
-        self.offset = offset
-        self.was_quantized = False
-
-    def to_quantized(self, group_size: int, bits: int) -> "MockCache":
-        quantized = MockCache(self.offset)
-        quantized.was_quantized = True
-        return quantized
-
-
-class TestSpeculativeDecodingLogic:
-    """Tests for the core speculative decoding logic."""
-
-    def test_draft_acceptance_identical_tokens(self) -> None:
-        """When draft matches verification, both should be accepted."""
-        # This tests the logic, not the full generator
-        draft_token = 42
-        verify_token = 42
-
-        accepted = draft_token == verify_token
-        assert accepted
-
-    def test_draft_rejection_different_tokens(self) -> None:
-        """When draft differs from verification, draft should be rejected."""
-        draft_token = 42
-        verify_token = 99
-
-        accepted = draft_token == verify_token
-        assert not accepted
-
-
-class TestMTPGenerationResponse:
-    """Tests for MTPGenerationResponse dataclass."""
-
-    def test_response_creation(self) -> None:
-        """Should create response with all fields."""
-        from exo.worker.engines.mlx.mtp.speculative_decode import MTPGenerationResponse
-
-        response = MTPGenerationResponse(
-            text="Hello",
-            token=42,
-            logprobs=mx.array([0.1, 0.2]),
-            from_draft=True,
-            prompt_tokens=10,
-            prompt_tps=100.0,
-            generation_tokens=5,
-            generation_tps=50.0,
-            peak_memory=1.5,
-            finish_reason=None,
-        )
-
-        assert response.text == "Hello"
-        assert response.token == 42
-        assert response.from_draft is True
-        assert response.finish_reason is None
-
-    def test_response_with_finish_reason(self) -> None:
-        """Should handle finish_reason."""
-        from exo.worker.engines.mlx.mtp.speculative_decode import MTPGenerationResponse
-
-        response = MTPGenerationResponse(
-            text="",
-            token=0,
-            logprobs=mx.array([0.0]),
-            from_draft=False,
-            prompt_tokens=10,
-            prompt_tps=100.0,
-            generation_tokens=100,
-            generation_tps=50.0,
-            peak_memory=1.5,
-            finish_reason="length",
-        )
-
-        assert response.finish_reason == "length"
-
-
-class TestIntegration:
-    """Integration tests for the full MTP pipeline."""
-
-    def test_mtp_module_with_mock_model(self) -> None:
-        """Test MTP module can be created and run with mock components."""
-        pytest.importorskip("mlx_lm.models.deepseek_v3")
-
-        from exo.worker.engines.mlx.mtp.module import MTPModule
-
-        # Create mock config
-        class MockConfig:
-            hidden_size = 64
-            intermediate_size = 128
-            num_attention_heads = 2
-            num_key_value_heads = 2
-            rms_norm_eps = 1e-6
-            q_lora_rank = None
-            kv_lora_rank = 32
-            qk_rope_head_dim = 8
-            v_head_dim = 16
-            qk_nope_head_dim = 16
-            rope_theta = 10000.0
-            rope_scaling = None
-            attention_bias = False
-            max_position_embeddings = 2048
-
-        config = MockConfig()
-        embedding = nn.Embedding(100, config.hidden_size)
-        lm_head = nn.Linear(config.hidden_size, 100, bias=False)
-        output_norm = nn.RMSNorm(config.hidden_size)
-
-        mtp = MTPModule(
-            config=config,  # type: ignore[arg-type]
-            shared_embedding=embedding,
-            shared_lm_head=lm_head,
-            output_norm=output_norm,
-        )
-
-        # Run forward pass
-        hidden = mx.random.normal((1, 1, config.hidden_size))
-        token = mx.array([[5]])
-
-        logits, new_hidden = mtp(hidden, token)
-
-        assert logits.shape == (1, 1, 100)
-        assert new_hidden.shape == (1, 1, config.hidden_size)
-        # Verify outputs are valid (not NaN)
-        assert not mx.any(mx.isnan(logits))
-        assert not mx.any(mx.isnan(new_hidden))

src/exo/worker/engines/mlx/utils_mlx.py

@@ -2,9 +2,7 @@ import json
 import os
 import resource
 import sys
-import threading
 import time
-from collections.abc import Callable
 from pathlib import Path
 from typing import Any, cast
 
@@ -22,13 +20,11 @@ except ImportError:
 
 from mlx_lm.models.cache import KVCache, QuantizedKVCache, RotatingKVCache
 from mlx_lm.models.deepseek_v3 import DeepseekV3Model
-from mlx_lm.models.gpt_oss import Model as GptOssModel
 from mlx_lm.tokenizer_utils import TokenizerWrapper
 
 from exo.worker.engines.mlx.constants import (
     CACHE_GROUP_SIZE,
     KV_CACHE_BITS,
-    MTP_ENABLED,
     TRUST_REMOTE_CODE,
 )
 
@@ -70,67 +66,6 @@ Group = mx.distributed.Group
 resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 4096))
 
 
-# MTP (Multi-Token Prediction) support for DeepSeek V3
-MTP_LAYER_INDEX = 61
-_original_deepseek_sanitize: Callable[..., dict[str, Any]] | None = None
-
-
-def _is_deepseek_v3_model(model: nn.Module) -> bool:
-    """Check if the model is DeepSeek V3."""
-    return hasattr(model, "model") and isinstance(model.model, DeepseekV3Model)
-
-
-def _patch_deepseek_sanitize_for_mtp() -> None:
-    """Patch DeepSeek V3 Model.sanitize to preserve MTP layer weights.
-
-    The original sanitize() method filters out layer 61 (MTP layer) weights.
-    This patch keeps them so we can extract and use the MTP module.
-    """
-    global _original_deepseek_sanitize
-    from mlx_lm.models.deepseek_v3 import Model as DeepSeekV3Model
-
-    if _original_deepseek_sanitize is not None:
-        # Already patched
-        return
-
-    _original_deepseek_sanitize = DeepSeekV3Model.sanitize
-
-    def sanitize_with_mtp(
-        self: DeepSeekV3Model, weights: dict[str, Any]
-    ) -> dict[str, Any]:
-        """Modified sanitize that keeps MTP layer weights."""
-        # First, call the original sanitize to handle all the weight transformations
-        # (dequantization, expert stacking, etc.)
-        if _original_deepseek_sanitize is None:
-            raise RuntimeError(
-                "_original_deepseek_sanitize is None - patch not applied correctly"
-            )
-        original_result: dict[str, Any] = _original_deepseek_sanitize(self, weights)
-
-        # Re-add the MTP layer weights that were filtered out
-        mtp_weights = {
-            k: v
-            for k, v in weights.items()
-            if k.startswith(f"model.layers.{MTP_LAYER_INDEX}")
-        }
-
-        return {**original_result, **mtp_weights}
-
-    DeepSeekV3Model.sanitize = sanitize_with_mtp
-
-
-def _restore_deepseek_sanitize() -> None:
-    """Restore the original DeepSeek V3 sanitize method."""
-    global _original_deepseek_sanitize
-    if _original_deepseek_sanitize is None:
-        return
-
-    from mlx_lm.models.deepseek_v3 import Model as DeepSeekV3Model
-
-    DeepSeekV3Model.sanitize = _original_deepseek_sanitize
-    _original_deepseek_sanitize = None
-
-
 # TODO: Test this
 #  ALSO https://github.com/exo-explore/exo/pull/233#discussion_r2549683673
 def get_weights_size(model_shard_meta: ShardMetadata) -> Memory:
@@ -146,45 +81,6 @@ def get_weights_size(model_shard_meta: ShardMetadata) -> Memory:
     )
 
 
-class ModelLoadingTimeoutError(Exception):
-    pass
-
-
-TimeoutCallback = Callable[[], None]
-
-
-def eval_with_timeout(
-    mlx_item: Any,  # pyright: ignore[reportAny]
-    timeout_seconds: float = 60.0,
-    on_timeout: TimeoutCallback | None = None,
-) -> None:
-    """Evaluate MLX item with a hard timeout.
-
-    If on_timeout callback is provided, it will be called before terminating
-    the process. This allows the runner to send a failure event before exit.
-    """
-    completed = threading.Event()
-
-    def watchdog() -> None:
-        if not completed.wait(timeout=timeout_seconds):
-            logger.error(
-                f"mlx_item evaluation timed out after {timeout_seconds:.0f}s. "
-                "This may indicate an issue with FAST_SYNCH and tensor parallel sharding. "
-                "Terminating process."
-            )
-            if on_timeout is not None:
-                on_timeout()
-            os._exit(1)
-
-    watchdog_thread = threading.Thread(target=watchdog, daemon=True)
-    watchdog_thread.start()
-
-    try:
-        mx.eval(mlx_item)  # pyright: ignore[reportAny]
-    finally:
-        completed.set()
-
-
 def mx_barrier(group: Group | None = None):
     mx.eval(
         mx.distributed.all_sum(
@@ -291,172 +187,34 @@ def initialize_mlx(
 
 
 def load_mlx_items(
-    bound_instance: BoundInstance,
-    group: Group | None,
-    on_timeout: TimeoutCallback | None = None,
+    bound_instance: BoundInstance, group: Group | None
 ) -> tuple[Model, TokenizerWrapper]:
-    """Load MLX model and tokenizer.
+    if group is None:
+        logger.info(f"Single device used for {bound_instance.instance}")
+        model_path = build_model_path(bound_instance.bound_shard.model_meta.model_id)
+        start_time = time.perf_counter()
+        model, _ = load_model(model_path, strict=True)
+        end_time = time.perf_counter()
+        logger.info(f"Time taken to load model: {(end_time - start_time):.2f}s")
+        tokenizer = get_tokenizer(model_path, bound_instance.bound_shard)
 
-    Returns:
-        Tuple of (model, tokenizer)
-    """
-    model_id = bound_instance.bound_shard.model_meta.model_id
-    mtp_module = None
-
-    # Patch sanitize for MTP if this might be DeepSeek V3
-    should_try_mtp = MTP_ENABLED and _might_be_deepseek_v3(model_id)
-    if should_try_mtp:
-        logger.info("Patching DeepSeek V3 sanitize for MTP weight preservation")
-        _patch_deepseek_sanitize_for_mtp()
-
-    try:
-        if group is None:
-            logger.info(f"Single device used for {bound_instance.instance}")
-            model_path = build_model_path(model_id)
-            start_time = time.perf_counter()
-            model, _ = load_model(model_path, strict=not should_try_mtp)
-            end_time = time.perf_counter()
-            logger.info(f"Time taken to load model: {(end_time - start_time):.2f}s")
-            tokenizer = get_tokenizer(model_path, bound_instance.bound_shard)
-
-        else:
-            logger.info("Starting distributed init")
-            start_time = time.perf_counter()
-            model, tokenizer = shard_and_load(
-                bound_instance.bound_shard, group=group, on_timeout=on_timeout
-            )
-            end_time = time.perf_counter()
-            logger.info(
-                f"Time taken to shard and load model: {(end_time - start_time):.2f}s"
-            )
-
-        # Extract MTP module if available
-        if should_try_mtp and _is_deepseek_v3_model(model):
-            mtp_module = _extract_mtp_module(model)
-            if mtp_module is not None:
-                logger.info("Successfully extracted MTP module from DeepSeek V3")
-
-    finally:
-        # Restore original sanitize
-        if should_try_mtp:
-            _restore_deepseek_sanitize()
+    else:
+        logger.info("Starting distributed init")
+        start_time = time.perf_counter()
+        model, tokenizer = shard_and_load(bound_instance.bound_shard, group=group)
+        end_time = time.perf_counter()
+        logger.info(
+            f"Time taken to shard and load model: {(end_time - start_time):.2f}s"
+        )
 
     set_wired_limit_for_model(get_weights_size(bound_instance.bound_shard))
 
-    # Store MTP module on the model for later access
-    if mtp_module is not None:
-        model.mtp_module = mtp_module  # noqa: B010
-
     return cast(Model, model), tokenizer
 
 
-def _might_be_deepseek_v3(model_id: str) -> bool:
-    """Check if model ID suggests this might be DeepSeek V3."""
-    model_id_lower = model_id.lower()
-    return "deepseek" in model_id_lower and (
-        "v3" in model_id_lower or "r1" in model_id_lower
-    )
-
-
-def _flatten_params(
-    params: dict[str, Any],
-    prefix: str = "",
-) -> dict[str, mx.array]:
-    """Flatten nested parameter dict to flat dict with dot-separated keys."""
-    result: dict[str, mx.array] = {}
-    for key, value in params.items():
-        full_key = f"{prefix}.{key}" if prefix else key
-        if isinstance(value, mx.array):
-            result[full_key] = value
-        elif isinstance(value, dict):
-            result.update(_flatten_params(value, full_key))
-    return result
-
-
-def _extract_mtp_module(model: nn.Module) -> Any | None:
-    """Extract MTP module from a loaded DeepSeek V3 model.
-
-    The MTP weights are stored in model.model.layers at index 61 (if preserved).
-    This function extracts them and creates an MTPModule.
-
-    Returns:
-        MTPModule if MTP weights were found and extracted, None otherwise.
-    """
-    from exo.worker.engines.mlx.mtp.module import (
-        MTPModule,
-        extract_mtp_weights,
-        load_mtp_weights_into_module,
-    )
-
-    try:
-        # Check if the model has the MTP layer
-        inner_model = getattr(model, "model", None)
-        if inner_model is None or not hasattr(inner_model, "layers"):
-            logger.debug("Model doesn't have expected structure for MTP extraction")
-            return None
-
-        layers: list[nn.Module] = inner_model.layers  # type: ignore[assignment]
-        if len(layers) <= MTP_LAYER_INDEX:
-            logger.debug(
-                f"Model has {len(layers)} layers, MTP layer {MTP_LAYER_INDEX} not found"
-            )
-            return None
-
-        # Get model config
-        config = getattr(model, "args", None)
-        if config is None:
-            logger.debug("Could not get model config for MTP module")
-            return None
-
-        # Create MTP module with shared weights
-        embed_tokens = getattr(inner_model, "embed_tokens", None)
-        lm_head = getattr(model, "lm_head", None)
-        norm = getattr(inner_model, "norm", None)
-
-        if embed_tokens is None or lm_head is None or norm is None:
-            logger.debug("Could not get required model components for MTP")
-            return None
-
-        mtp_module = MTPModule(
-            config=config,
-            shared_embedding=embed_tokens,
-            shared_lm_head=lm_head,
-            output_norm=norm,
-        )
-
-        # Extract MTP layer weights from the model's parameters
-        # The weights should be at model.model.layers.61.*
-        # model.parameters() returns a nested dict, we need to flatten it
-        raw_params: dict[str, Any] = dict(model.parameters())  # type: ignore[arg-type]
-        model_weights = _flatten_params(raw_params)
-        mtp_weights = extract_mtp_weights(model_weights)
-
-        if not mtp_weights:
-            logger.debug("No MTP weights found in model parameters")
-            return None
-
-        # Load weights into MTP module
-        load_mtp_weights_into_module(mtp_module, mtp_weights)
-
-        # Remove MTP layer from main model to avoid double computation
-        # Create new layers list without the MTP layer
-        new_layers = [layer for i, layer in enumerate(layers) if i != MTP_LAYER_INDEX]
-        inner_model.layers = new_layers  # noqa: B010
-
-        logger.info(
-            f"Extracted MTP module, main model now has {len(new_layers)} layers"
-        )
-        return mtp_module
-
-    except Exception as e:
-        logger.warning(f"Failed to extract MTP module: {e}")
-        return None
-
-
 def shard_and_load(
     shard_metadata: ShardMetadata,
     group: Group,
-    on_timeout: TimeoutCallback | None = None,
 ) -> tuple[nn.Module, TokenizerWrapper]:
     model_path = build_model_path(shard_metadata.model_meta.model_id)
 
@@ -493,15 +251,7 @@ def shard_and_load(
             logger.info(f"loading model from {model_path} with pipeline parallelism")
             model = pipeline_auto_parallel(model, group, shard_metadata)
 
-    # Estimate timeout based on model size
-    base_timeout = float(os.environ.get("EXO_MODEL_LOAD_TIMEOUT", "60"))
-    model_size_gb = get_weights_size(shard_metadata).in_bytes / (1024**3)
-    timeout_seconds = base_timeout + model_size_gb / 5
-    logger.info(
-        f"Evaluating model parameters with timeout of {timeout_seconds:.0f}s "
-        f"(model size: {model_size_gb:.1f}GB)"
-    )
-    eval_with_timeout(model.parameters(), timeout_seconds, on_timeout)
+    mx.eval(model.parameters())
 
     # TODO: Do we need this?
     mx.eval(model)
@@ -615,8 +365,6 @@ def apply_chat_template(
         tools=chat_task_data.tools,
     )
 
-    logger.info(prompt)
-
     return prompt
 
 
@@ -648,11 +396,6 @@ def make_kv_cache(
 ) -> list[KVCache | RotatingKVCache | QuantizedKVCache]:
     assert hasattr(model, "layers")
 
-    # TODO: Do this for all models
-    if hasattr(model, "make_cache") and isinstance(model, GptOssModel):
-        logger.info("Using MLX LM's make cache")
-        return model.make_cache()  # type: ignore
-
     if max_kv_size is None:
         if KV_CACHE_BITS is None:
             logger.info("Using default KV cache")

src/exo/worker/runner/bootstrap.py

@@ -17,23 +17,15 @@ def entrypoint(
     task_receiver: MpReceiver[Task],
     _logger: "loguru.Logger",
 ) -> None:
-    fast_synch_override = os.environ.get("EXO_FAST_SYNCH")
-    if fast_synch_override == "on" or (
-        fast_synch_override != "off"
-        and (
-            isinstance(bound_instance.instance, MlxJacclInstance)
-            and len(bound_instance.instance.ibv_devices) >= 2
-        )
+    if (
+        isinstance(bound_instance.instance, MlxJacclInstance)
+        and len(bound_instance.instance.ibv_devices) >= 2
     ):
         os.environ["MLX_METAL_FAST_SYNCH"] = "1"
-    else:
-        os.environ["MLX_METAL_FAST_SYNCH"] = "0"
 
     global logger
     logger = _logger
 
-    logger.info(f"Fast synch flag: {os.environ['MLX_METAL_FAST_SYNCH']}")
-
     # Import main after setting global logger - this lets us just import logger from this module
     try:
         from exo.worker.runner.runner import main

src/exo/worker/runner/runner.py

@@ -1,21 +1,9 @@
 import time
-from collections.abc import Generator
-from contextlib import contextmanager
-from functools import cache
-from typing import cast
 
 import mlx.core as mx
-from mlx_lm.models.gpt_oss import Model as GptOssModel
-from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
-    HarmonyEncodingName,
-    Role,
-    StreamableParser,
-    load_harmony_encoding,
-)
 
 from exo.shared.types.api import ChatCompletionMessageText
 from exo.shared.types.chunks import TokenChunk
-from exo.shared.types.common import CommandId
 from exo.shared.types.events import (
     ChunkGenerated,
     Event,
@@ -23,7 +11,6 @@ from exo.shared.types.events import (
     TaskAcknowledged,
     TaskStatusUpdated,
 )
-from exo.shared.types.models import ModelId
 from exo.shared.types.tasks import (
     ChatCompletion,
     ConnectToGroup,
@@ -52,7 +39,6 @@ from exo.shared.types.worker.runners import (
     RunnerWarmingUp,
 )
 from exo.utils.channels import MpReceiver, MpSender
-from exo.worker.engines.mlx import Model
 from exo.worker.engines.mlx.generator.generate import mlx_generate, warmup_inference
 from exo.worker.engines.mlx.utils_mlx import (
     initialize_mlx,
@@ -62,33 +48,6 @@ from exo.worker.engines.mlx.utils_mlx import (
 from exo.worker.runner.bootstrap import logger
 
 
-@contextmanager
-def send_error_chunk_on_exception(
-    event_sender: MpSender[Event],
-    command_id: CommandId,
-    model_id: ModelId,
-    device_rank: int,
-):
-    try:
-        yield
-    except Exception as e:
-        logger.error(e)
-        if device_rank == 0:
-            event_sender.send(
-                ChunkGenerated(
-                    command_id=command_id,
-                    chunk=TokenChunk(
-                        idx=0,
-                        model=model_id,
-                        text="",
-                        token_id=0,
-                        finish_reason="error",
-                        error_message=str(e),
-                    ),
-                )
-            )
-
-
 def main(
     bound_instance: BoundInstance,
     event_sender: MpSender[Event],
@@ -150,20 +109,7 @@ def main(
                         )
                     )
 
-                    def on_model_load_timeout() -> None:
-                        event_sender.send(
-                            RunnerStatusUpdated(
-                                runner_id=runner_id,
-                                runner_status=RunnerFailed(
-                                    error_message="Model loading timed out"
-                                ),
-                            )
-                        )
-                        time.sleep(0.5)
-
-                    model, tokenizer = load_mlx_items(
-                        bound_instance, group, on_timeout=on_model_load_timeout
-                    )
+                    model, tokenizer = load_mlx_items(bound_instance, group)
 
                     current_status = RunnerLoaded()
                     logger.info("runner loaded")
@@ -180,7 +126,7 @@ def main(
 
                     logger.info(f"warming up inference for instance: {instance}")
                     toks = warmup_inference(
-                        model=cast(Model, model),
+                        model=model,
                         tokenizer=tokenizer,
                         # kv_prefix_cache=kv_prefix_cache,  # supply for warmup-time prefix caching
                     )
@@ -193,6 +139,8 @@ def main(
                 case ChatCompletion(task_params=task_params, command_id=command_id) if (
                     isinstance(current_status, RunnerReady)
                 ):
+                    assert model
+                    assert tokenizer
                     logger.info(f"received chat request: {str(task)[:500]}")
                     current_status = RunnerRunning()
                     logger.info("runner running")
@@ -201,47 +149,33 @@ def main(
                             runner_id=runner_id, runner_status=current_status
                         )
                     )
-                    with send_error_chunk_on_exception(
-                        event_sender,
-                        command_id,
-                        shard_metadata.model_meta.model_id,
-                        shard_metadata.device_rank,
+                    assert task_params.messages[0].content is not None
+                    _check_for_debug_prompts(task_params.messages[0].content)
+
+                    # Generate responses using the actual MLX generation
+                    for response in mlx_generate(
+                        model=model,
+                        tokenizer=tokenizer,
+                        task=task_params,
                     ):
-                        assert model
-                        assert tokenizer
-                        assert task_params.messages[0].content is not None
-                        _check_for_debug_prompts(task_params.messages[0].content)
-
-                        # Generate responses using the actual MLX generation
-                        mlx_generator = mlx_generate(
-                            model=cast(Model, model),
-                            tokenizer=tokenizer,
-                            task=task_params,
-                        )
-
-                        # GPT-OSS specific parsing to match other model formats.
-                        if isinstance(model, GptOssModel):
-                            mlx_generator = parse_gpt_oss(mlx_generator)
-
-                        # TODO: Add tool call parser here
-
-                        for response in mlx_generator:
-                            match response:
-                                case GenerationResponse():
-                                    if shard_metadata.device_rank == 0:
-                                        event_sender.send(
-                                            ChunkGenerated(
-                                                command_id=command_id,
-                                                chunk=TokenChunk(
-                                                    idx=response.token,
-                                                    model=shard_metadata.model_meta.model_id,
-                                                    text=response.text,
-                                                    token_id=response.token,
-                                                    finish_reason=response.finish_reason,
-                                                    stats=response.stats,
-                                                ),
-                                            )
+                        match response:
+                            case GenerationResponse():
+                                if shard_metadata.device_rank == 0:
+                                    event_sender.send(
+                                        ChunkGenerated(
+                                            command_id=command_id,
+                                            chunk=TokenChunk(
+                                                idx=response.token,
+                                                model=shard_metadata.model_meta.model_id,
+                                                text=response.text,
+                                                token_id=response.token,
+                                                finish_reason=response.finish_reason,
+                                                stats=response.stats,
+                                            ),
                                         )
+                                    )
+                                # case TokenizedResponse():
+                                # TODO: something here ig
 
                     current_status = RunnerReady()
                     logger.info("runner ready")
@@ -273,43 +207,6 @@ def main(
                 break
 
 
-@cache
-def get_gpt_oss_encoding():
-    encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
-    return encoding
-
-
-def parse_gpt_oss(
-    responses: Generator[GenerationResponse],
-) -> Generator[GenerationResponse]:
-    encoding = get_gpt_oss_encoding()
-    stream = StreamableParser(encoding, role=Role.ASSISTANT)
-    thinking = False
-
-    for response in responses:
-        stream.process(response.token)
-
-        delta = stream.last_content_delta
-        ch = stream.current_channel
-
-        if ch == "analysis" and not thinking:
-            thinking = True
-            yield response.model_copy(update={"text": "<think>"})
-
-        if ch != "analysis" and thinking:
-            thinking = False
-            yield response.model_copy(update={"text": "</think>"})
-
-        if delta:
-            yield response.model_copy(update={"text": delta})
-
-        if response.finish_reason is not None:
-            if thinking:
-                yield response.model_copy(update={"text": "</think>"})
-            yield response
-            break
-
-
 EXO_RUNNER_MUST_FAIL = "EXO RUNNER MUST FAIL"
 EXO_RUNNER_MUST_OOM = "EXO RUNNER MUST OOM"
 EXO_RUNNER_MUST_TIMEOUT = "EXO RUNNER MUST TIMEOUT"

src/exo/worker/tests/unittests/test_runner/test_error_handling.py

@@ -1,50 +0,0 @@
-# pyright: reportAny=false
-from unittest.mock import MagicMock
-
-from exo.shared.types.chunks import TokenChunk
-from exo.shared.types.common import CommandId
-from exo.shared.types.events import ChunkGenerated
-from exo.worker.runner.runner import send_error_chunk_on_exception
-from exo.worker.tests.constants import MODEL_A_ID
-
-
-def test_send_error_chunk_on_exception_no_error() -> None:
-    event_sender = MagicMock()
-    command_id = CommandId()
-
-    with send_error_chunk_on_exception(
-        event_sender, command_id, MODEL_A_ID, device_rank=0
-    ):
-        _ = 1 + 1
-
-    event_sender.send.assert_not_called()
-
-
-def test_send_error_chunk_on_exception_catches_error() -> None:
-    event_sender = MagicMock()
-    command_id = CommandId()
-
-    with send_error_chunk_on_exception(
-        event_sender, command_id, MODEL_A_ID, device_rank=0
-    ):
-        raise ValueError("test error")
-
-    event_sender.send.assert_called_once()
-    call_args = event_sender.send.call_args[0][0]
-    assert isinstance(call_args, ChunkGenerated)
-    assert call_args.command_id == command_id
-    assert isinstance(call_args.chunk, TokenChunk)
-    assert call_args.chunk.finish_reason == "error"
-    assert call_args.chunk.error_message == "test error"
-
-
-def test_send_error_chunk_on_exception_skips_non_rank_zero() -> None:
-    event_sender = MagicMock()
-    command_id = CommandId()
-
-    with send_error_chunk_on_exception(
-        event_sender, command_id, MODEL_A_ID, device_rank=1
-    ):
-        raise ValueError("test error")
-
-    event_sender.send.assert_not_called()

src/exo/worker/utils/net_profile.py

@@ -1,64 +1,49 @@
-import anyio
-import httpx
-from anyio import create_task_group
+import http.client
+
+from anyio import create_task_group, to_thread
 from loguru import logger
 
 from exo.shared.topology import Topology
 from exo.shared.types.common import NodeId
 
-REACHABILITY_ATTEMPTS = 3
-
 
 async def check_reachability(
     target_ip: str,
     expected_node_id: NodeId,
+    self_node_id: NodeId,
     out: dict[NodeId, set[str]],
-    client: httpx.AsyncClient,
 ) -> None:
     """Check if a node is reachable at the given IP and verify its identity."""
-    if ":" in target_ip:
-        # TODO: use real IpAddress types
-        target_ip = f"[{target_ip}]"
-    url = f"http://{target_ip}:52415/node_id"
 
-    remote_node_id = None
-    last_error = None
-
-    for _ in range(REACHABILITY_ATTEMPTS):
+    def _fetch_remote_node_id() -> NodeId | None:
+        connection = http.client.HTTPConnection(target_ip, 52415, timeout=1)
         try:
-            r = await client.get(url)
-            if r.status_code != 200:
-                await anyio.sleep(1)
-                continue
+            connection.request("GET", "/node_id")
+            response = connection.getresponse()
+            if response.status != 200:
+                return None
 
-            body = r.text.strip().strip('"')
-            if not body:
-                await anyio.sleep(1)
-                continue
+            body = response.read().decode("utf-8").strip()
 
-            remote_node_id = NodeId(body)
-            break
+            # Strip quotes if present (JSON string response)
+            if body.startswith('"') and body.endswith('"') and len(body) >= 2:
+                body = body[1:-1]
 
-        # expected failure cases
-        except (
-            httpx.TimeoutException,
-            httpx.NetworkError,
-        ):
-            await anyio.sleep(1)
-
-        # other failures should be logged on last attempt
-        except httpx.HTTPError as e:
-            last_error = e
-            await anyio.sleep(1)
-
-    if last_error is not None:
-        logger.warning(
-            f"connect error {type(last_error).__name__} from {target_ip} after {REACHABILITY_ATTEMPTS} attempts; treating as down"
-        )
+            return NodeId(body) or None
+        except OSError:
+            return None
+        except http.client.HTTPException:
+            return None
+        finally:
+            connection.close()
 
+    remote_node_id = await to_thread.run_sync(_fetch_remote_node_id)
     if remote_node_id is None:
         return
 
+    if remote_node_id == self_node_id:
+        return
+
     if remote_node_id != expected_node_id:
         logger.warning(
             f"Discovered node with unexpected node_id; "
@@ -76,33 +61,18 @@ async def check_reachable(
     topology: Topology, self_node_id: NodeId
 ) -> dict[NodeId, set[str]]:
     """Check which nodes are reachable and return their IPs."""
-
     reachable: dict[NodeId, set[str]] = {}
-
-    # these are intentionally httpx's defaults so we can tune them later
-    timeout = httpx.Timeout(timeout=5.0)
-    limits = httpx.Limits(
-        max_connections=100,
-        max_keepalive_connections=20,
-        keepalive_expiry=5,
-    )
-
-    async with (
-        httpx.AsyncClient(timeout=timeout, limits=limits) as client,
-        create_task_group() as tg,
-    ):
+    async with create_task_group() as tg:
         for node in topology.list_nodes():
             if not node.node_profile:
                 continue
-            if node.node_id == self_node_id:
-                continue
             for iface in node.node_profile.network_interfaces:
                 tg.start_soon(
                     check_reachability,
                     iface.ip_address,
                     node.node_id,
+                    self_node_id,
                     reachable,
-                    client,
                 )
 
     return reachable

uv.lock

@@ -236,7 +236,6 @@ dependencies = [
     { name = "exo-pyo3-bindings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "hypercorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "loguru", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -248,6 +247,7 @@ dependencies = [
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rustworkx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "tiktoken", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+    { name = "tomlkit", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "types-aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
 
@@ -269,7 +269,6 @@ requires-dist = [
     { name = "exo-pyo3-bindings", editable = "rust/exo_pyo3_bindings" },
     { name = "fastapi", specifier = ">=0.116.1" },
     { name = "filelock", specifier = ">=3.18.0" },
-    { name = "httpx", specifier = ">=0.28.1" },
     { name = "huggingface-hub", specifier = ">=0.33.4" },
     { name = "hypercorn", specifier = ">=0.18.0" },
     { name = "loguru", specifier = ">=0.7.3" },
@@ -281,6 +280,7 @@ requires-dist = [
     { name = "pydantic", specifier = ">=2.11.7" },
     { name = "rustworkx", specifier = ">=0.17.1" },
     { name = "tiktoken", specifier = ">=0.12.0" },
+    { name = "tomlkit", specifier = ">=0.14.0" },
     { name = "types-aiofiles", specifier = ">=24.1.0.20250708" },
 ]
 
@@ -1380,6 +1380,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" },
 ]
 
+[[package]]
+name = "tomlkit"
+version = "0.14.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" },
+]
+
 [[package]]
 name = "tqdm"
 version = "4.67.1"