Fix draft release detection (attempt 3) (#1176)

## Motivation

Previous fix still failed in CI. Suspecting permissions issue with
GITHUB_TOKEN not being able to see draft releases via API.

## Changes

1. Add explicit `permissions: contents: write` to the job
2. Use `gh release list` first to check if draft exists (this uses a
different code path that might work better)
3. Add debug echo statements

## Test Plan

Delete v1.0.63 tag and re-push after merging.

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

.github/workflows/build-app.yml

@@ -1,5 +1,16 @@
 name: Build EXO macOS DMG
 
+# Release workflow:
+# 1. Create a draft GitHub Release with the tag name (e.g. v1.0.0) and write release notes in markdown
+# 2. Push the tag: git tag v1.0.0 && git push origin v1.0.0
+# 3. This workflow builds, signs, and notarizes the DMG
+# 4. Release notes are embedded in appcast.xml for Sparkle (rendered as markdown)
+# 5. DMG and appcast.xml are uploaded to S3
+# 6. The draft GitHub Release is published with the DMG attached
+#
+# For alpha releases (e.g. v1.0.0-alpha.1): draft release and notes are optional.
+# If no draft exists, a release is auto-created with generated notes.
+
 on:
   workflow_dispatch:
   push:
@@ -11,8 +22,10 @@ on:
 jobs:
   build-macos-app:
     runs-on: "macos-26"
+    permissions:
+      contents: write
     env:
-      SPARKLE_VERSION: 2.8.1
+      SPARKLE_VERSION: 2.9.0-beta.1
       SPARKLE_DOWNLOAD_PREFIX: ${{ secrets.SPARKLE_DOWNLOAD_PREFIX }}
       SPARKLE_FEED_URL: ${{ secrets.SPARKLE_FEED_URL }}
       SPARKLE_ED25519_PUBLIC: ${{ secrets.SPARKLE_ED25519_PUBLIC }}
@@ -87,6 +100,52 @@ jobs:
             exit 1
           fi
 
+      - name: Fetch and validate release notes
+        if: github.ref_type == 'tag'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Find draft release by name using gh release list (more reliable with default token)
+          echo "Looking for draft release named '$GITHUB_REF_NAME'..."
+          DRAFT_EXISTS=$(gh release list --json name,isDraft --jq ".[] | select(.isDraft == true) | select(.name == \"$GITHUB_REF_NAME\") | .name" 2>/dev/null || echo "")
+
+          if [[ -z "$DRAFT_EXISTS" ]]; then
+            if [[ "$IS_ALPHA" == "true" ]]; then
+              echo "No draft release found for alpha tag $GITHUB_REF_NAME (optional for alphas)"
+              echo "HAS_RELEASE_NOTES=false" >> $GITHUB_ENV
+              exit 0
+            fi
+            echo "ERROR: No draft release found for tag $GITHUB_REF_NAME"
+            echo "Please create a draft release with release notes before pushing the tag."
+            exit 1
+          fi
+
+          # Fetch full release details via API to get body and ID
+          echo "Found draft release, fetching details..."
+          RELEASE_JSON=$(gh api repos/${{ github.repository }}/releases --jq ".[] | select(.draft == true) | select(.name == \"$GITHUB_REF_NAME\")" 2>/dev/null || echo "")
+
+          # Extract release notes
+          NOTES=$(echo "$RELEASE_JSON" | jq -r '.body // ""')
+          if [[ -z "$NOTES" || "$NOTES" == "null" ]]; then
+            if [[ "$IS_ALPHA" == "true" ]]; then
+              echo "Draft release has no notes (optional for alphas)"
+              echo "HAS_RELEASE_NOTES=false" >> $GITHUB_ENV
+              exit 0
+            fi
+            echo "ERROR: Draft release exists but has no release notes"
+            echo "Please add release notes to the draft release before pushing the tag."
+            exit 1
+          fi
+
+          # Save release ID for later publishing
+          RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
+          echo "DRAFT_RELEASE_ID=$RELEASE_ID" >> $GITHUB_ENV
+          echo "HAS_RELEASE_NOTES=true" >> $GITHUB_ENV
+
+          echo "Found draft release (ID: $RELEASE_ID), saving release notes..."
+          echo "$NOTES" > /tmp/release_notes.md
+          echo "RELEASE_NOTES_FILE=/tmp/release_notes.md" >> $GITHUB_ENV
+
       # ============================================================
       # Install dependencies
       # ============================================================
@@ -304,6 +363,28 @@ jobs:
             $CHANNEL_FLAG \
             .
 
+      - name: Inject release notes into appcast
+        if: github.ref_type == 'tag' && env.HAS_RELEASE_NOTES == 'true'
+        env:
+          RELEASE_VERSION: ${{ env.RELEASE_VERSION }}
+        run: |
+          # Inject markdown release notes with sparkle:format="markdown" (Sparkle 2.9+)
+          export NOTES=$(cat "$RELEASE_NOTES_FILE")
+
+          # Insert description after the enclosure tag for this version
+          awk '
+            /<enclosure[^>]*>/ && index($0, ENVIRON["RELEASE_VERSION"]) {
+              print
+              print "            <description sparkle:format=\"markdown\"><![CDATA["
+              print ENVIRON["NOTES"]
+              print "            ]]></description>"
+              next
+            }
+            { print }
+          ' output/appcast.xml > output/appcast.xml.tmp && mv output/appcast.xml.tmp output/appcast.xml
+
+          echo "Injected markdown release notes for version $RELEASE_VERSION"
+
       # ============================================================
       # Upload artifacts
       # ============================================================
@@ -336,3 +417,26 @@ jobs:
             aws s3 cp "$DMG_NAME" "s3://${SPARKLE_S3_BUCKET}/${PREFIX}EXO-latest.dmg"
             aws s3 cp appcast.xml "s3://${SPARKLE_S3_BUCKET}/${PREFIX}appcast.xml" --content-type application/xml --cache-control no-cache
           fi
+
+      - name: Publish GitHub Release
+        if: github.ref_type == 'tag'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          DMG_PATH="output/EXO-${RELEASE_VERSION}.dmg"
+
+          if [[ "$HAS_RELEASE_NOTES" == "true" ]]; then
+            # Update the draft release with the tag and upload DMG
+            gh api --method PATCH "repos/${{ github.repository }}/releases/$DRAFT_RELEASE_ID" \
+              -f tag_name="$GITHUB_REF_NAME" \
+              -F draft=false
+            gh release upload "$GITHUB_REF_NAME" "$DMG_PATH" --clobber
+            echo "Published release $GITHUB_REF_NAME with DMG attached"
+          else
+            # Alpha without draft release - create one with auto-generated notes
+            gh release create "$GITHUB_REF_NAME" "$DMG_PATH" \
+              --title "$GITHUB_REF_NAME" \
+              --generate-notes \
+              --prerelease
+            echo "Created alpha release $GITHUB_REF_NAME with auto-generated notes"
+          fi

app/EXO/EXO.xcodeproj/project.pbxproj

@@ -585,7 +585,7 @@
 			repositoryURL = "https://github.com/sparkle-project/Sparkle.git";
 			requirement = {
 				kind = upToNextMajorVersion;
-				minimumVersion = 2.8.1;
+				minimumVersion = 2.9.0-beta.1;
 			};
 		};
 /* End XCRemoteSwiftPackageReference section */

app/EXO/EXO.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -6,8 +6,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/sparkle-project/Sparkle.git",
       "state" : {
-        "revision" : "5581748cef2bae787496fe6d61139aebe0a451f6",
-        "version" : "2.8.1"
+        "revision" : "e641adb41915a8409895e2e30666aa64e487b637",
+        "version" : "2.9.0-beta.1"
       }
     }
   ],

app/EXO/EXO/ContentView.swift

@@ -56,6 +56,11 @@ struct ContentView: View {
     }
 
     private var shouldShowLocalNetworkWarning: Bool {
+        // Show warning if local network is not working and EXO is running.
+        // The checker uses a longer timeout on first launch to allow time for
+        // the permission prompt, so this correctly handles both:
+        // 1. User denied permission on first launch
+        // 2. Permission broke after restart (macOS TCC bug)
         if case .notWorking = localNetworkChecker.status {
             return controller.status != .stopped
         }

app/EXO/EXO/Services/LocalNetworkChecker.swift

@@ -5,8 +5,8 @@ import os.log
 /// Checks if the app's local network permission is actually functional.
 ///
 /// macOS local network permission can appear enabled in System Preferences but not
-/// actually work after a restart. This service detects this by creating a UDP
-/// connection to the mDNS multicast address (224.0.0.251:5353).
+/// actually work after a restart. This service uses NWConnection to mDNS multicast
+/// to verify actual connectivity.
 @MainActor
 final class LocalNetworkChecker: ObservableObject {
     enum Status: Equatable {
@@ -35,30 +35,43 @@ final class LocalNetworkChecker: ObservableObject {
     }
 
     private static let logger = Logger(subsystem: "io.exo.EXO", category: "LocalNetworkChecker")
+    private static let hasCompletedInitialCheckKey = "LocalNetworkChecker.hasCompletedInitialCheck"
 
     @Published private(set) var status: Status = .unknown
-    @Published private(set) var lastConnectionState: String = "none"
 
     private var connection: NWConnection?
     private var checkTask: Task<Void, Never>?
 
+    /// Whether we've completed at least one check (stored in UserDefaults)
+    private var hasCompletedInitialCheck: Bool {
+        get { UserDefaults.standard.bool(forKey: Self.hasCompletedInitialCheckKey) }
+        set { UserDefaults.standard.set(newValue, forKey: Self.hasCompletedInitialCheckKey) }
+    }
+
     /// Checks if local network access is working.
     func check() {
         checkTask?.cancel()
         status = .checking
-        lastConnectionState = "connecting"
+
+        // Use longer timeout on first launch to allow time for permission prompt
+        let isFirstCheck = !hasCompletedInitialCheck
+        let timeout: UInt64 = isFirstCheck ? 30_000_000_000 : 3_000_000_000
 
         checkTask = Task { [weak self] in
             guard let self else { return }
-            let result = await self.performCheck()
+
+            Self.logger.info("Checking local network connectivity (first check: \(isFirstCheck))")
+            let result = await self.checkConnectivity(timeout: timeout)
             self.status = result
+            self.hasCompletedInitialCheck = true
+
             Self.logger.info("Local network check complete: \(result.displayText)")
         }
     }
 
-    private func performCheck() async -> Status {
-        Self.logger.info("Checking local network access via UDP multicast")
-
+    /// Checks connectivity using NWConnection to mDNS multicast.
+    /// The connection attempt triggers the permission prompt if not yet shown.
+    private func checkConnectivity(timeout: UInt64) async -> Status {
         connection?.cancel()
         connection = nil
 
@@ -84,22 +97,7 @@ final class LocalNetworkChecker: ObservableObject {
                 continuation.resume(returning: status)
             }
 
-            conn.stateUpdateHandler = { [weak self] state in
-                let stateStr: String
-                switch state {
-                case .setup: stateStr = "setup"
-                case .preparing: stateStr = "preparing"
-                case .ready: stateStr = "ready"
-                case .waiting(let e): stateStr = "waiting(\(e))"
-                case .failed(let e): stateStr = "failed(\(e))"
-                case .cancelled: stateStr = "cancelled"
-                @unknown default: stateStr = "unknown"
-                }
-
-                Task { @MainActor in
-                    self?.lastConnectionState = stateStr
-                }
-
+            conn.stateUpdateHandler = { state in
                 switch state {
                 case .ready:
                     resumeOnce(.working)
@@ -108,6 +106,7 @@ final class LocalNetworkChecker: ObservableObject {
                     if errorStr.contains("54") || errorStr.contains("ECONNRESET") {
                         resumeOnce(.notWorking(reason: "Connection blocked"))
                     }
+                // Otherwise keep waiting - might be showing permission prompt
                 case .failed(let error):
                     let errorStr = "\(error)"
                     if errorStr.contains("65") || errorStr.contains("EHOSTUNREACH")
@@ -127,7 +126,7 @@ final class LocalNetworkChecker: ObservableObject {
             conn.start(queue: .main)
 
             Task {
-                try? await Task.sleep(nanoseconds: 3_000_000_000)
+                try? await Task.sleep(nanoseconds: timeout)
                 let state = conn.state
                 switch state {
                 case .ready:

bench/exo_bench.py

@@ -241,6 +241,9 @@ class PromptSizer:
             ids = tokenizer.apply_chat_template(
                 messages, tokenize=True, add_generation_prompt=True
             )
+            # Fix for transformers 5.x
+            if hasattr(ids, "input_ids"):
+                ids = ids.input_ids
             return int(len(ids))
 
         return count_fn

dashboard/src/lib/components/ChatForm.svelte

@@ -60,12 +60,39 @@
 		return models;
 	});
 
-	// Auto-select the first available model if none is selected
+	// Track previous model IDs to detect newly added models (plain variable to avoid reactive loop)
+	let previousModelIds: Set<string> = new Set();
+
+	// Auto-select the first available model if none is selected, if current selection is stale, or if a new model is added
 	$effect(() => {
 		const models = availableModels();
-		if (models.length > 0 && !currentModel) {
-			setSelectedChatModel(models[0].id);
+		const currentModelIds = new Set(models.map(m => m.id));
+
+		if (models.length > 0) {
+			// Find newly added models (in current but not in previous)
+			const newModels = models.filter(m => !previousModelIds.has(m.id));
+
+			// If no model selected, select the first available
+			if (!currentModel) {
+				setSelectedChatModel(models[0].id);
+			}
+			// If current model is stale (no longer has a running instance), reset to first available
+			else if (!models.some(m => m.id === currentModel)) {
+				setSelectedChatModel(models[0].id);
+			}
+			// If a new model was just added, select it
+			else if (newModels.length > 0 && previousModelIds.size > 0) {
+				setSelectedChatModel(newModels[0].id);
+			}
+		} else {
+			// No instances running - clear the selected model
+			if (currentModel) {
+				setSelectedChatModel('');
+			}
 		}
+
+		// Update previous model IDs for next comparison
+		previousModelIds = currentModelIds;
 	});
 
 	function getInstanceModelId(instanceWrapped: unknown): string {

dashboard/src/routes/+page.svelte

@@ -400,10 +400,8 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 				const errorText = await response.text();
 				console.error('Failed to launch instance:', errorText);
 			} else {
-				// Auto-select the launched model only if no model is currently selected
-				if (!selectedChatModel()) {
-					setSelectedChatModel(modelId);
-				}
+				// Always auto-select the newly launched model so the user chats to what they just launched
+				setSelectedChatModel(modelId);
 				
 				// Scroll to the bottom of instances container to show the new instance
 				// Use multiple attempts to ensure DOM has updated with the new instance
@@ -763,6 +761,10 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 	async function deleteInstance(instanceId: string) {
 		if (!confirm(`Delete instance ${instanceId.slice(0, 8)}...?`)) return;
 		
+		// Get the model ID of the instance being deleted before we delete it
+		const deletedInstanceModelId = getInstanceModelId(instanceData[instanceId]);
+		const wasSelected = selectedChatModel() === deletedInstanceModelId;
+		
 		try {
 			const response = await fetch(`/instance/${instanceId}`, {
 				method: 'DELETE',
@@ -771,6 +773,24 @@ function toggleInstanceDownloadDetails(nodeId: string): void {
 			
 			if (!response.ok) {
 				console.error('Failed to delete instance:', response.status);
+			} else if (wasSelected) {
+				// If we deleted the currently selected model, switch to another available model
+				// Find another instance that isn't the one we just deleted
+				const remainingInstances = Object.entries(instanceData).filter(([id]) => id !== instanceId);
+				if (remainingInstances.length > 0) {
+					// Select the last instance (most recently added, since objects preserve insertion order)
+					const [, lastInstance] = remainingInstances[remainingInstances.length - 1];
+					const newModelId = getInstanceModelId(lastInstance);
+					if (newModelId && newModelId !== 'Unknown' && newModelId !== 'Unknown Model') {
+						setSelectedChatModel(newModelId);
+					} else {
+						// Clear selection if no valid model found
+						setSelectedChatModel('');
+					}
+				} else {
+					// No more instances, clear the selection
+					setSelectedChatModel('');
+				}
 			}
 		} catch (error) {
 			console.error('Error deleting instance:', error);

justfile

@@ -1,3 +1,5 @@
+export NIX_CONFIG := "extra-experimental-features = nix-command flakes"
+
 fmt:
     nix fmt
 

pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
     "tiktoken>=0.12.0", # required for kimi k2 tokenizer
     "hypercorn>=0.18.0",
     "openai-harmony>=0.0.8",
-    "tomlkit>=0.14.0",
+    "httpx>=0.28.1",
 ]
 
 [project.scripts]

resources/model_cards/deepseek-v3.1-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "deepseek-v3.1-4bit"
-model_id = "mlx-community/DeepSeek-V3.1-4bit"
-name = "DeepSeek V3.1 (4-bit)"
-description = "DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/DeepSeek-V3.1-4bit"
-pretty_name = "DeepSeek V3.1 (4-bit)"
-n_layers = 61
-hidden_size = 7168
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 405874409472

resources/model_cards/deepseek-v3.1-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "deepseek-v3.1-8bit"
-model_id = "mlx-community/DeepSeek-V3.1-8bit"
-name = "DeepSeek V3.1 (8-bit)"
-description = "DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/DeepSeek-V3.1-8bit"
-pretty_name = "DeepSeek V3.1 (8-bit)"
-n_layers = 61
-hidden_size = 7168
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 765577920512

resources/model_cards/glm-4.5-air-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "glm-4.5-air-8bit"
-model_id = "mlx-community/GLM-4.5-Air-8bit"
-name = "GLM 4.5 Air 8bit"
-description = "GLM 4.5 Air 8bit"
-tags = []
-
-[metadata]
-model_id = "mlx-community/GLM-4.5-Air-8bit"
-pretty_name = "GLM 4.5 Air 8bit"
-n_layers = 46
-hidden_size = 4096
-supports_tensor = false
-
-[metadata.storage_size]
-in_bytes = 122406567936

resources/model_cards/glm-4.5-air-bf16.toml

@@ -1,15 +0,0 @@
-short_id = "glm-4.5-air-bf16"
-model_id = "mlx-community/GLM-4.5-Air-bf16"
-name = "GLM 4.5 Air bf16"
-description = "GLM 4.5 Air bf16"
-tags = []
-
-[metadata]
-model_id = "mlx-community/GLM-4.5-Air-bf16"
-pretty_name = "GLM 4.5 Air bf16"
-n_layers = 46
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 229780750336

resources/model_cards/glm-4.7-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "glm-4.7-4bit"
-model_id = "mlx-community/GLM-4.7-4bit"
-name = "GLM 4.7 4bit"
-description = "GLM 4.7 4bit"
-tags = []
-
-[metadata]
-model_id = "mlx-community/GLM-4.7-4bit"
-pretty_name = "GLM 4.7 4bit"
-n_layers = 91
-hidden_size = 5120
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 198556925568

resources/model_cards/glm-4.7-6bit.toml

@@ -1,15 +0,0 @@
-short_id = "glm-4.7-6bit"
-model_id = "mlx-community/GLM-4.7-6bit"
-name = "GLM 4.7 6bit"
-description = "GLM 4.7 6bit"
-tags = []
-
-[metadata]
-model_id = "mlx-community/GLM-4.7-6bit"
-pretty_name = "GLM 4.7 6bit"
-n_layers = 91
-hidden_size = 5120
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 286737579648

resources/model_cards/glm-4.7-8bit-gs32.toml

@@ -1,15 +0,0 @@
-short_id = "glm-4.7-8bit-gs32"
-model_id = "mlx-community/GLM-4.7-8bit-gs32"
-name = "GLM 4.7 8bit (gs32)"
-description = "GLM 4.7 8bit (gs32)"
-tags = []
-
-[metadata]
-model_id = "mlx-community/GLM-4.7-8bit-gs32"
-pretty_name = "GLM 4.7 8bit (gs32)"
-n_layers = 91
-hidden_size = 5120
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 396963397248

resources/model_cards/gpt-oss-120b-MXFP4-Q8.toml

@@ -1,15 +0,0 @@
-short_id = "gpt-oss-120b-MXFP4-Q8"
-model_id = "mlx-community/gpt-oss-120b-MXFP4-Q8"
-name = "GPT-OSS 120B (MXFP4-Q8, MLX)"
-description = "OpenAI's GPT-OSS 120B is a 117B-parameter Mixture-of-Experts model designed for high-reasoning and general-purpose use; this variant is a 4-bit MLX conversion for Apple Silicon."
-tags = []
-
-[metadata]
-model_id = "mlx-community/gpt-oss-120b-MXFP4-Q8"
-pretty_name = "GPT-OSS 120B (MXFP4-Q8, MLX)"
-n_layers = 36
-hidden_size = 2880
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 70652212224

resources/model_cards/gpt-oss-20b-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "gpt-oss-20b-4bit"
-model_id = "mlx-community/gpt-oss-20b-MXFP4-Q4"
-name = "GPT-OSS 20B (MXFP4-Q4, MLX)"
-description = "OpenAI's GPT-OSS 20B is a medium-sized MoE model for lower-latency and local or specialized use cases; this MLX variant uses MXFP4 4-bit quantization."
-tags = []
-
-[metadata]
-model_id = "mlx-community/gpt-oss-20b-MXFP4-Q4"
-pretty_name = "GPT-OSS 20B (MXFP4-Q4, MLX)"
-n_layers = 24
-hidden_size = 2880
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 12025908224

resources/model_cards/kimi-k2-instruct-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "kimi-k2-instruct-4bit"
-model_id = "mlx-community/Kimi-K2-Instruct-4bit"
-name = "Kimi K2 Instruct (4-bit)"
-description = "Kimi K2 is a large language model trained on the Kimi K2 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Kimi-K2-Instruct-4bit"
-pretty_name = "Kimi K2 Instruct (4-bit)"
-n_layers = 61
-hidden_size = 7168
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 620622774272

resources/model_cards/kimi-k2-thinking.toml

@@ -1,15 +0,0 @@
-short_id = "kimi-k2-thinking"
-model_id = "mlx-community/Kimi-K2-Thinking"
-name = "Kimi K2 Thinking (4-bit)"
-description = "Kimi K2 Thinking is the latest, most capable version of open-source thinking model."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Kimi-K2-Thinking"
-pretty_name = "Kimi K2 Thinking (4-bit)"
-n_layers = 61
-hidden_size = 7168
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 706522120192

resources/model_cards/llama-3.1-70b.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.1-70b"
-model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
-name = "Llama 3.1 70B (4-bit)"
-description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
-pretty_name = "Llama 3.1 70B (4-bit)"
-n_layers = 80
-hidden_size = 8192
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 40652242944

resources/model_cards/llama-3.1-8b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.1-8b-8bit"
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
-name = "Llama 3.1 8B (8-bit)"
-description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
-pretty_name = "Llama 3.1 8B (8-bit)"
-n_layers = 32
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 8954839040

resources/model_cards/llama-3.1-8b-bf16.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.1-8b-bf16"
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
-name = "Llama 3.1 8B (BF16)"
-description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
-pretty_name = "Llama 3.1 8B (BF16)"
-n_layers = 32
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 16882073600

resources/model_cards/llama-3.1-8b.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.1-8b"
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
-name = "Llama 3.1 8B (4-bit)"
-description = "Llama 3.1 is a large language model trained on the Llama 3.1 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
-pretty_name = "Llama 3.1 8B (4-bit)"
-n_layers = 32
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 4637851648

resources/model_cards/llama-3.2-1b.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.2-1b"
-model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
-name = "Llama 3.2 1B (4-bit)"
-description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
-pretty_name = "Llama 3.2 1B (4-bit)"
-n_layers = 16
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 729808896

resources/model_cards/llama-3.2-3b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.2-3b-8bit"
-model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
-name = "Llama 3.2 3B (8-bit)"
-description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
-pretty_name = "Llama 3.2 3B (8-bit)"
-n_layers = 28
-hidden_size = 3072
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 3501195264

resources/model_cards/llama-3.2-3b.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.2-3b"
-model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
-name = "Llama 3.2 3B (4-bit)"
-description = "Llama 3.2 is a large language model trained on the Llama 3.2 dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
-pretty_name = "Llama 3.2 3B (4-bit)"
-n_layers = 28
-hidden_size = 3072
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 1863319552

resources/model_cards/llama-3.3-70b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.3-70b-8bit"
-model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
-name = "Llama 3.3 70B (8-bit)"
-description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
-pretty_name = "Llama 3.3 70B (8-bit)"
-n_layers = 80
-hidden_size = 8192
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 76799803392

resources/model_cards/llama-3.3-70b-fp16.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.3-70b-fp16"
-model_id = "mlx-community/llama-3.3-70b-instruct-fp16"
-name = "Llama 3.3 70B (FP16)"
-description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
-tags = []
-
-[metadata]
-model_id = "mlx-community/llama-3.3-70b-instruct-fp16"
-pretty_name = "Llama 3.3 70B (FP16)"
-n_layers = 80
-hidden_size = 8192
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 144383672320

resources/model_cards/llama-3.3-70b.toml

@@ -1,15 +0,0 @@
-short_id = "llama-3.3-70b"
-model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
-name = "Llama 3.3 70B (4-bit)"
-description = "The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
-pretty_name = "Llama 3.3 70B"
-n_layers = 80
-hidden_size = 8192
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 40652242944

resources/model_cards/minimax-m2.1-3bit.toml

@@ -1,15 +0,0 @@
-short_id = "minimax-m2.1-3bit"
-model_id = "mlx-community/MiniMax-M2.1-3bit"
-name = "MiniMax M2.1 3bit"
-description = "MiniMax M2.1 3bit"
-tags = []
-
-[metadata]
-model_id = "mlx-community/MiniMax-M2.1-3bit"
-pretty_name = "MiniMax M2.1 3bit"
-n_layers = 61
-hidden_size = 3072
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 100086644736

resources/model_cards/minimax-m2.1-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "minimax-m2.1-8bit"
-model_id = "mlx-community/MiniMax-M2.1-8bit"
-name = "MiniMax M2.1 8bit"
-description = "MiniMax M2.1 8bit"
-tags = []
-
-[metadata]
-model_id = "mlx-community/MiniMax-M2.1-8bit"
-pretty_name = "MiniMax M2.1 8bit"
-n_layers = 61
-hidden_size = 3072
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 242986745856

resources/model_cards/qwen3-0.6b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-0.6b-8bit"
-model_id = "mlx-community/Qwen3-0.6B-8bit"
-name = "Qwen3 0.6B (8-bit)"
-description = "Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-0.6B-8bit"
-pretty_name = "Qwen3 0.6B (8-bit)"
-n_layers = 28
-hidden_size = 1024
-supports_tensor = false
-
-[metadata.storage_size]
-in_bytes = 698351616

resources/model_cards/qwen3-0.6b.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-0.6b"
-model_id = "mlx-community/Qwen3-0.6B-4bit"
-name = "Qwen3 0.6B (4-bit)"
-description = "Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-0.6B-4bit"
-pretty_name = "Qwen3 0.6B (4-bit)"
-n_layers = 28
-hidden_size = 1024
-supports_tensor = false
-
-[metadata.storage_size]
-in_bytes = 342884352

resources/model_cards/qwen3-235b-a22b-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-235b-a22b-4bit"
-model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
-name = "Qwen3 235B A22B (4-bit)"
-description = "Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
-pretty_name = "Qwen3 235B A22B (4-bit)"
-n_layers = 94
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 141733920768

resources/model_cards/qwen3-235b-a22b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-235b-a22b-8bit"
-model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
-name = "Qwen3 235B A22B (8-bit)"
-description = "Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
-pretty_name = "Qwen3 235B A22B (8-bit)"
-n_layers = 94
-hidden_size = 4096
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 268435456000

resources/model_cards/qwen3-30b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-30b-8bit"
-model_id = "mlx-community/Qwen3-30B-A3B-8bit"
-name = "Qwen3 30B A3B (8-bit)"
-description = "Qwen3 30B is a large language model trained on the Qwen3 30B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-30B-A3B-8bit"
-pretty_name = "Qwen3 30B A3B (8-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 33279705088

resources/model_cards/qwen3-30b.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-30b"
-model_id = "mlx-community/Qwen3-30B-A3B-4bit"
-name = "Qwen3 30B A3B (4-bit)"
-description = "Qwen3 30B is a large language model trained on the Qwen3 30B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-30B-A3B-4bit"
-pretty_name = "Qwen3 30B A3B (4-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 17612931072

resources/model_cards/qwen3-80b-a3B-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-80b-a3B-4bit"
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
-name = "Qwen3 80B A3B (4-bit)"
-description = "Qwen3 80B"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
-pretty_name = "Qwen3 80B A3B (4-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 46976204800

resources/model_cards/qwen3-80b-a3B-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-80b-a3B-8bit"
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
-name = "Qwen3 80B A3B (8-bit)"
-description = "Qwen3 80B"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
-pretty_name = "Qwen3 80B A3B (8-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 88814387200

resources/model_cards/qwen3-80b-a3B-thinking-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-80b-a3B-thinking-4bit"
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
-name = "Qwen3 80B A3B Thinking (4-bit)"
-description = "Qwen3 80B Reasoning model"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
-pretty_name = "Qwen3 80B A3B (4-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 88814387200

resources/model_cards/qwen3-80b-a3B-thinking-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-80b-a3B-thinking-8bit"
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
-name = "Qwen3 80B A3B Thinking (8-bit)"
-description = "Qwen3 80B Reasoning model"
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
-pretty_name = "Qwen3 80B A3B (8-bit)"
-n_layers = 48
-hidden_size = 2048
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 88814387200

resources/model_cards/qwen3-coder-480b-a35b-4bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-coder-480b-a35b-4bit"
-model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
-name = "Qwen3 Coder 480B A35B (4-bit)"
-description = "Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
-pretty_name = "Qwen3 Coder 480B A35B (4-bit)"
-n_layers = 62
-hidden_size = 6144
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 289910292480

resources/model_cards/qwen3-coder-480b-a35b-8bit.toml

@@ -1,15 +0,0 @@
-short_id = "qwen3-coder-480b-a35b-8bit"
-model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
-name = "Qwen3 Coder 480B A35B (8-bit)"
-description = "Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset."
-tags = []
-
-[metadata]
-model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
-pretty_name = "Qwen3 Coder 480B A35B (8-bit)"
-n_layers = 62
-hidden_size = 6144
-supports_tensor = true
-
-[metadata.storage_size]
-in_bytes = 579820584960

src/exo/master/api.py

@@ -13,12 +13,6 @@ from hypercorn.asyncio import serve  # pyright: ignore[reportUnknownVariableType
 from hypercorn.config import Config
 from hypercorn.typing import ASGIFramework
 from loguru import logger
-from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
-    HarmonyEncodingName,
-    Role,
-    StreamableParser,
-    load_harmony_encoding,
-)
 
 from exo.master.placement import place_instance as get_instance_placements
 from exo.shared.apply import apply
@@ -67,8 +61,6 @@ from exo.utils.channels import Receiver, Sender, channel
 from exo.utils.dashboard_path import find_dashboard
 from exo.utils.event_buffer import OrderedBuffer
 
-encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
-
 
 def chunk_to_response(
     chunk: TokenChunk, command_id: CommandId
@@ -381,35 +373,8 @@ class API:
             instance_id=instance_id,
         )
 
-    async def _process_gpt_oss(self, token_chunks: Receiver[TokenChunk]):
-        stream = StreamableParser(encoding, role=Role.ASSISTANT)
-        thinking = False
-
-        async for chunk in token_chunks:
-            stream.process(chunk.token_id)
-
-            delta = stream.last_content_delta
-            ch = stream.current_channel
-
-            if ch == "analysis" and not thinking:
-                thinking = True
-                yield chunk.model_copy(update={"text": "<think>"})
-
-            if ch != "analysis" and thinking:
-                thinking = False
-                yield chunk.model_copy(update={"text": "</think>"})
-
-            if delta:
-                yield chunk.model_copy(update={"text": delta})
-
-            if chunk.finish_reason is not None:
-                if thinking:
-                    yield chunk.model_copy(update={"text": "</think>"})
-                yield chunk
-                break
-
     async def _chat_chunk_stream(
-        self, command_id: CommandId, parse_gpt_oss: bool
+        self, command_id: CommandId
     ) -> AsyncGenerator[TokenChunk, None]:
         """Yield `TokenChunk`s for a given command until completion."""
 
@@ -417,16 +382,10 @@ class API:
             self._chat_completion_queues[command_id], recv = channel[TokenChunk]()
 
             with recv as token_chunks:
-                if parse_gpt_oss:
-                    async for chunk in self._process_gpt_oss(token_chunks):
-                        yield chunk
-                        if chunk.finish_reason is not None:
-                            break
-                else:
-                    async for chunk in token_chunks:
-                        yield chunk
-                        if chunk.finish_reason is not None:
-                            break
+                async for chunk in token_chunks:
+                    yield chunk
+                    if chunk.finish_reason is not None:
+                        break
 
         except anyio.get_cancelled_exc_class():
             # TODO: TaskCancelled
@@ -442,11 +401,11 @@ class API:
             del self._chat_completion_queues[command_id]
 
     async def _generate_chat_stream(
-        self, command_id: CommandId, parse_gpt_oss: bool
+        self, command_id: CommandId
     ) -> AsyncGenerator[str, None]:
         """Generate chat completion stream as JSON strings."""
 
-        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
+        async for chunk in self._chat_chunk_stream(command_id):
             chunk_response: ChatCompletionResponse = chunk_to_response(
                 chunk, command_id
             )
@@ -458,7 +417,7 @@ class API:
                 yield "data: [DONE]\n\n"
 
     async def _collect_chat_completion(
-        self, command_id: CommandId, parse_gpt_oss: bool
+        self, command_id: CommandId
     ) -> ChatCompletionResponse:
         """Collect all token chunks for a chat completion and return a single response."""
 
@@ -466,7 +425,7 @@ class API:
         model: str | None = None
         finish_reason: FinishReason | None = None
 
-        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
+        async for chunk in self._chat_chunk_stream(command_id):
             if model is None:
                 model = chunk.model
 
@@ -495,7 +454,7 @@ class API:
         )
 
     async def _collect_chat_completion_with_stats(
-        self, command_id: CommandId, parse_gpt_oss: bool
+        self, command_id: CommandId
     ) -> BenchChatCompletionResponse:
         text_parts: list[str] = []
         model: str | None = None
@@ -503,7 +462,7 @@ class API:
 
         stats: GenerationStats | None = None
 
-        async for chunk in self._chat_chunk_stream(command_id, parse_gpt_oss):
+        async for chunk in self._chat_chunk_stream(command_id):
             if model is None:
                 model = chunk.model
 
@@ -544,8 +503,6 @@ class API:
         """Handle chat completions, supporting both streaming and non-streaming responses."""
         model_meta = await resolve_model_meta(payload.model)
         payload.model = model_meta.model_id
-        parse_gpt_oss = "gpt-oss" in model_meta.model_id.lower()
-        logger.info(f"{parse_gpt_oss=}")
 
         if not any(
             instance.shard_assignments.model_id == payload.model
@@ -562,17 +519,16 @@ class API:
         await self._send(command)
         if payload.stream:
             return StreamingResponse(
-                self._generate_chat_stream(command.command_id, parse_gpt_oss),
+                self._generate_chat_stream(command.command_id),
                 media_type="text/event-stream",
             )
 
-        return await self._collect_chat_completion(command.command_id, parse_gpt_oss)
+        return await self._collect_chat_completion(command.command_id)
 
     async def bench_chat_completions(
         self, payload: BenchChatCompletionTaskParams
     ) -> BenchChatCompletionResponse:
         model_meta = await resolve_model_meta(payload.model)
-        parse_gpt_oss = "gpt-oss" in model_meta.model_id.lower()
         payload.model = model_meta.model_id
 
         if not any(
@@ -589,10 +545,7 @@ class API:
         command = ChatCompletion(request_params=payload)
         await self._send(command)
 
-        response = await self._collect_chat_completion_with_stats(
-            command.command_id,
-            parse_gpt_oss,
-        )
+        response = await self._collect_chat_completion_with_stats(command.command_id)
         return response
 
     def _calculate_total_available_memory(self) -> Memory:

src/exo/shared/logging.py

@@ -29,6 +29,11 @@ class _InterceptHandler(logging.Handler):
 
 def logger_setup(log_file: Path | None, verbosity: int = 0):
     """Set up logging for this process - formatting, file handles, verbosity and output"""
+
+    logging.getLogger("exo_pyo3_bindings").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+
     logger.remove()
 
     # replace all stdlib loggers with _InterceptHandlers that log to loguru

src/exo/shared/models/model_cards.py

@@ -1,8 +1,5 @@
-from anyio import Path, open_file
-import tomlkit
-
+from exo.shared.types.memory import Memory
 from exo.shared.types.models import ModelId, ModelMetadata
-from exo.shared.models.model_meta import get_model_meta
 from exo.utils.pydantic_ext import CamelCaseModel
 
 
@@ -14,27 +11,542 @@ class ModelCard(CamelCaseModel):
     tags: list[str]
     metadata: ModelMetadata
 
-    @staticmethod
-    async def load(path: Path) -> "ModelCard":
-        async with await open_file(path) as f:
-            data = await f.read()
-            py = tomlkit.loads(data)
-            return ModelCard.model_validate(py)
 
-    async def save(self, path: Path):
-        async with await open_file(path, "w") as f:
-            py = self.model_dump()
-            data = tomlkit.dumps(py)  # pyright: ignore[reportUnknownMemberType]
-            await f.write(data)
-
-    @staticmethod
-    async def from_hf(model_id: str) -> "ModelCard":
-        short_name = model_id.split("/")[-1]
-        return ModelCard(
-            short_id=short_name,
-            model_id=ModelId(model_id),
-            name=short_name,
-            description=f"Custom model from {model_id}",
-            tags=[],
-            metadata=await get_model_meta(model_id),
-        )
+MODEL_CARDS: dict[str, ModelCard] = {
+    # deepseek v3
+    "deepseek-v3.1-4bit": ModelCard(
+        short_id="deepseek-v3.1-4bit",
+        model_id=ModelId("mlx-community/DeepSeek-V3.1-4bit"),
+        name="DeepSeek V3.1 (4-bit)",
+        description="""DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/DeepSeek-V3.1-4bit"),
+            pretty_name="DeepSeek V3.1 (4-bit)",
+            storage_size=Memory.from_gb(378),
+            n_layers=61,
+            hidden_size=7168,
+            supports_tensor=True,
+        ),
+    ),
+    "deepseek-v3.1-8bit": ModelCard(
+        short_id="deepseek-v3.1-8bit",
+        model_id=ModelId("mlx-community/DeepSeek-V3.1-8bit"),
+        name="DeepSeek V3.1 (8-bit)",
+        description="""DeepSeek V3.1 is a large language model trained on the DeepSeek V3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/DeepSeek-V3.1-8bit"),
+            pretty_name="DeepSeek V3.1 (8-bit)",
+            storage_size=Memory.from_gb(713),
+            n_layers=61,
+            hidden_size=7168,
+            supports_tensor=True,
+        ),
+    ),
+    # kimi k2
+    "kimi-k2-instruct-4bit": ModelCard(
+        short_id="kimi-k2-instruct-4bit",
+        model_id=ModelId("mlx-community/Kimi-K2-Instruct-4bit"),
+        name="Kimi K2 Instruct (4-bit)",
+        description="""Kimi K2 is a large language model trained on the Kimi K2 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Kimi-K2-Instruct-4bit"),
+            pretty_name="Kimi K2 Instruct (4-bit)",
+            storage_size=Memory.from_gb(578),
+            n_layers=61,
+            hidden_size=7168,
+            supports_tensor=True,
+        ),
+    ),
+    "kimi-k2-thinking": ModelCard(
+        short_id="kimi-k2-thinking",
+        model_id=ModelId("mlx-community/Kimi-K2-Thinking"),
+        name="Kimi K2 Thinking (4-bit)",
+        description="""Kimi K2 Thinking is the latest, most capable version of open-source thinking model.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Kimi-K2-Thinking"),
+            pretty_name="Kimi K2 Thinking (4-bit)",
+            storage_size=Memory.from_gb(658),
+            n_layers=61,
+            hidden_size=7168,
+            supports_tensor=True,
+        ),
+    ),
+    # llama-3.1
+    "llama-3.1-8b": ModelCard(
+        short_id="llama-3.1-8b",
+        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"),
+        name="Llama 3.1 8B (4-bit)",
+        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"),
+            pretty_name="Llama 3.1 8B (4-bit)",
+            storage_size=Memory.from_mb(4423),
+            n_layers=32,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.1-8b-8bit": ModelCard(
+        short_id="llama-3.1-8b-8bit",
+        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"),
+        name="Llama 3.1 8B (8-bit)",
+        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"),
+            pretty_name="Llama 3.1 8B (8-bit)",
+            storage_size=Memory.from_mb(8540),
+            n_layers=32,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.1-8b-bf16": ModelCard(
+        short_id="llama-3.1-8b-bf16",
+        model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"),
+        name="Llama 3.1 8B (BF16)",
+        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"),
+            pretty_name="Llama 3.1 8B (BF16)",
+            storage_size=Memory.from_mb(16100),
+            n_layers=32,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.1-70b": ModelCard(
+        short_id="llama-3.1-70b",
+        model_id=ModelId("mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"),
+        name="Llama 3.1 70B (4-bit)",
+        description="""Llama 3.1 is a large language model trained on the Llama 3.1 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"),
+            pretty_name="Llama 3.1 70B (4-bit)",
+            storage_size=Memory.from_mb(38769),
+            n_layers=80,
+            hidden_size=8192,
+            supports_tensor=True,
+        ),
+    ),
+    # llama-3.2
+    "llama-3.2-1b": ModelCard(
+        short_id="llama-3.2-1b",
+        model_id=ModelId("mlx-community/Llama-3.2-1B-Instruct-4bit"),
+        name="Llama 3.2 1B (4-bit)",
+        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Llama-3.2-1B-Instruct-4bit"),
+            pretty_name="Llama 3.2 1B (4-bit)",
+            storage_size=Memory.from_mb(696),
+            n_layers=16,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.2-3b": ModelCard(
+        short_id="llama-3.2-3b",
+        model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-4bit"),
+        name="Llama 3.2 3B (4-bit)",
+        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-4bit"),
+            pretty_name="Llama 3.2 3B (4-bit)",
+            storage_size=Memory.from_mb(1777),
+            n_layers=28,
+            hidden_size=3072,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.2-3b-8bit": ModelCard(
+        short_id="llama-3.2-3b-8bit",
+        model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-8bit"),
+        name="Llama 3.2 3B (8-bit)",
+        description="""Llama 3.2 is a large language model trained on the Llama 3.2 dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Llama-3.2-3B-Instruct-8bit"),
+            pretty_name="Llama 3.2 3B (8-bit)",
+            storage_size=Memory.from_mb(3339),
+            n_layers=28,
+            hidden_size=3072,
+            supports_tensor=True,
+        ),
+    ),
+    # llama-3.3
+    "llama-3.3-70b": ModelCard(
+        short_id="llama-3.3-70b",
+        model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-4bit"),
+        name="Llama 3.3 70B (4-bit)",
+        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-4bit"),
+            pretty_name="Llama 3.3 70B",
+            storage_size=Memory.from_mb(38769),
+            n_layers=80,
+            hidden_size=8192,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.3-70b-8bit": ModelCard(
+        short_id="llama-3.3-70b-8bit",
+        model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-8bit"),
+        name="Llama 3.3 70B (8-bit)",
+        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Llama-3.3-70B-Instruct-8bit"),
+            pretty_name="Llama 3.3 70B (8-bit)",
+            storage_size=Memory.from_mb(73242),
+            n_layers=80,
+            hidden_size=8192,
+            supports_tensor=True,
+        ),
+    ),
+    "llama-3.3-70b-fp16": ModelCard(
+        short_id="llama-3.3-70b-fp16",
+        model_id=ModelId("mlx-community/llama-3.3-70b-instruct-fp16"),
+        name="Llama 3.3 70B (FP16)",
+        description="""The Meta Llama 3.3 multilingual large language model (LLM) is an instruction tuned generative model in 70B (text in/text out)""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/llama-3.3-70b-instruct-fp16"),
+            pretty_name="Llama 3.3 70B (FP16)",
+            storage_size=Memory.from_mb(137695),
+            n_layers=80,
+            hidden_size=8192,
+            supports_tensor=True,
+        ),
+    ),
+    # qwen3
+    "qwen3-0.6b": ModelCard(
+        short_id="qwen3-0.6b",
+        model_id=ModelId("mlx-community/Qwen3-0.6B-4bit"),
+        name="Qwen3 0.6B (4-bit)",
+        description="""Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-0.6B-4bit"),
+            pretty_name="Qwen3 0.6B (4-bit)",
+            storage_size=Memory.from_mb(327),
+            n_layers=28,
+            hidden_size=1024,
+            supports_tensor=False,
+        ),
+    ),
+    "qwen3-0.6b-8bit": ModelCard(
+        short_id="qwen3-0.6b-8bit",
+        model_id=ModelId("mlx-community/Qwen3-0.6B-8bit"),
+        name="Qwen3 0.6B (8-bit)",
+        description="""Qwen3 0.6B is a large language model trained on the Qwen3 0.6B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-0.6B-8bit"),
+            pretty_name="Qwen3 0.6B (8-bit)",
+            storage_size=Memory.from_mb(666),
+            n_layers=28,
+            hidden_size=1024,
+            supports_tensor=False,
+        ),
+    ),
+    "qwen3-30b": ModelCard(
+        short_id="qwen3-30b",
+        model_id=ModelId("mlx-community/Qwen3-30B-A3B-4bit"),
+        name="Qwen3 30B A3B (4-bit)",
+        description="""Qwen3 30B is a large language model trained on the Qwen3 30B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-30B-A3B-4bit"),
+            pretty_name="Qwen3 30B A3B (4-bit)",
+            storage_size=Memory.from_mb(16797),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-30b-8bit": ModelCard(
+        short_id="qwen3-30b-8bit",
+        model_id=ModelId("mlx-community/Qwen3-30B-A3B-8bit"),
+        name="Qwen3 30B A3B (8-bit)",
+        description="""Qwen3 30B is a large language model trained on the Qwen3 30B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-30B-A3B-8bit"),
+            pretty_name="Qwen3 30B A3B (8-bit)",
+            storage_size=Memory.from_mb(31738),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-80b-a3B-4bit": ModelCard(
+        short_id="qwen3-80b-a3B-4bit",
+        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"),
+        name="Qwen3 80B A3B (4-bit)",
+        description="""Qwen3 80B""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"),
+            pretty_name="Qwen3 80B A3B (4-bit)",
+            storage_size=Memory.from_mb(44800),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-80b-a3B-8bit": ModelCard(
+        short_id="qwen3-80b-a3B-8bit",
+        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"),
+        name="Qwen3 80B A3B (8-bit)",
+        description="""Qwen3 80B""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"),
+            pretty_name="Qwen3 80B A3B (8-bit)",
+            storage_size=Memory.from_mb(84700),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-80b-a3B-thinking-4bit": ModelCard(
+        short_id="qwen3-80b-a3B-thinking-4bit",
+        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"),
+        name="Qwen3 80B A3B Thinking (4-bit)",
+        description="""Qwen3 80B Reasoning model""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"),
+            pretty_name="Qwen3 80B A3B (4-bit)",
+            storage_size=Memory.from_mb(84700),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-80b-a3B-thinking-8bit": ModelCard(
+        short_id="qwen3-80b-a3B-thinking-8bit",
+        model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"),
+        name="Qwen3 80B A3B Thinking (8-bit)",
+        description="""Qwen3 80B Reasoning model""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"),
+            pretty_name="Qwen3 80B A3B (8-bit)",
+            storage_size=Memory.from_mb(84700),
+            n_layers=48,
+            hidden_size=2048,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-235b-a22b-4bit": ModelCard(
+        short_id="qwen3-235b-a22b-4bit",
+        model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"),
+        name="Qwen3 235B A22B (4-bit)",
+        description="""Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"),
+            pretty_name="Qwen3 235B A22B (4-bit)",
+            storage_size=Memory.from_gb(132),
+            n_layers=94,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-235b-a22b-8bit": ModelCard(
+        short_id="qwen3-235b-a22b-8bit",
+        model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"),
+        name="Qwen3 235B A22B (8-bit)",
+        description="""Qwen3 235B (Active 22B) is a large language model trained on the Qwen3 235B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"),
+            pretty_name="Qwen3 235B A22B (8-bit)",
+            storage_size=Memory.from_gb(250),
+            n_layers=94,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-coder-480b-a35b-4bit": ModelCard(
+        short_id="qwen3-coder-480b-a35b-4bit",
+        model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"),
+        name="Qwen3 Coder 480B A35B (4-bit)",
+        description="""Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"),
+            pretty_name="Qwen3 Coder 480B A35B (4-bit)",
+            storage_size=Memory.from_gb(270),
+            n_layers=62,
+            hidden_size=6144,
+            supports_tensor=True,
+        ),
+    ),
+    "qwen3-coder-480b-a35b-8bit": ModelCard(
+        short_id="qwen3-coder-480b-a35b-8bit",
+        model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"),
+        name="Qwen3 Coder 480B A35B (8-bit)",
+        description="""Qwen3 Coder 480B (Active 35B) is a large language model trained on the Qwen3 Coder 480B dataset.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"),
+            pretty_name="Qwen3 Coder 480B A35B (8-bit)",
+            storage_size=Memory.from_gb(540),
+            n_layers=62,
+            hidden_size=6144,
+            supports_tensor=True,
+        ),
+    ),
+    # gpt-oss
+    "gpt-oss-120b-MXFP4-Q8": ModelCard(
+        short_id="gpt-oss-120b-MXFP4-Q8",
+        model_id=ModelId("mlx-community/gpt-oss-120b-MXFP4-Q8"),
+        name="GPT-OSS 120B (MXFP4-Q8, MLX)",
+        description="""OpenAI's GPT-OSS 120B is a 117B-parameter Mixture-of-Experts model designed for high-reasoning and general-purpose use; this variant is a 4-bit MLX conversion for Apple Silicon.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/gpt-oss-120b-MXFP4-Q8"),
+            pretty_name="GPT-OSS 120B (MXFP4-Q8, MLX)",
+            storage_size=Memory.from_kb(68_996_301),
+            n_layers=36,
+            hidden_size=2880,
+            supports_tensor=True,
+        ),
+    ),
+    "gpt-oss-20b-MXFP4-Q8": ModelCard(
+        short_id="gpt-oss-20b-MXFP4-Q8",
+        model_id=ModelId("mlx-community/gpt-oss-20b-MXFP4-Q8"),
+        name="GPT-OSS 20B (MXFP4-Q8, MLX)",
+        description="""OpenAI's GPT-OSS 20B is a medium-sized MoE model for lower-latency and local or specialized use cases; this variant is a 4-bit MLX conversion for Apple Silicon.""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/gpt-oss-20b-MXFP4-Q8"),
+            pretty_name="GPT-OSS 20B (MXFP4-Q8, MLX)",
+            storage_size=Memory.from_kb(11_744_051),
+            n_layers=24,
+            hidden_size=2880,
+            supports_tensor=True,
+        ),
+    ),
+    # glm 4.5
+    "glm-4.5-air-8bit": ModelCard(
+        # Needs to be quantized g32 or g16 to work with tensor parallel
+        short_id="glm-4.5-air-8bit",
+        model_id=ModelId("mlx-community/GLM-4.5-Air-8bit"),
+        name="GLM 4.5 Air 8bit",
+        description="""GLM 4.5 Air 8bit""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/GLM-4.5-Air-8bit"),
+            pretty_name="GLM 4.5 Air 8bit",
+            storage_size=Memory.from_gb(114),
+            n_layers=46,
+            hidden_size=4096,
+            supports_tensor=False,
+        ),
+    ),
+    "glm-4.5-air-bf16": ModelCard(
+        short_id="glm-4.5-air-bf16",
+        model_id=ModelId("mlx-community/GLM-4.5-Air-bf16"),
+        name="GLM 4.5 Air bf16",
+        description="""GLM 4.5 Air bf16""",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/GLM-4.5-Air-bf16"),
+            pretty_name="GLM 4.5 Air bf16",
+            storage_size=Memory.from_gb(214),
+            n_layers=46,
+            hidden_size=4096,
+            supports_tensor=True,
+        ),
+    ),
+    # glm 4.7
+    "glm-4.7-4bit": ModelCard(
+        short_id="glm-4.7-4bit",
+        model_id=ModelId("mlx-community/GLM-4.7-4bit"),
+        name="GLM 4.7 4bit",
+        description="GLM 4.7 4bit",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/GLM-4.7-4bit"),
+            pretty_name="GLM 4.7 4bit",
+            storage_size=Memory.from_bytes(198556925568),
+            n_layers=91,
+            hidden_size=5120,
+            supports_tensor=True,
+        ),
+    ),
+    "glm-4.7-6bit": ModelCard(
+        short_id="glm-4.7-6bit",
+        model_id=ModelId("mlx-community/GLM-4.7-6bit"),
+        name="GLM 4.7 6bit",
+        description="GLM 4.7 6bit",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/GLM-4.7-6bit"),
+            pretty_name="GLM 4.7 6bit",
+            storage_size=Memory.from_bytes(286737579648),
+            n_layers=91,
+            hidden_size=5120,
+            supports_tensor=True,
+        ),
+    ),
+    "glm-4.7-8bit-gs32": ModelCard(
+        short_id="glm-4.7-8bit-gs32",
+        model_id=ModelId("mlx-community/GLM-4.7-8bit-gs32"),
+        name="GLM 4.7 8bit (gs32)",
+        description="GLM 4.7 8bit (gs32)",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/GLM-4.7-8bit-gs32"),
+            pretty_name="GLM 4.7 8bit (gs32)",
+            storage_size=Memory.from_bytes(396963397248),
+            n_layers=91,
+            hidden_size=5120,
+            supports_tensor=True,
+        ),
+    ),
+    # minimax-m2
+    "minimax-m2.1-8bit": ModelCard(
+        short_id="minimax-m2.1-8bit",
+        model_id=ModelId("mlx-community/MiniMax-M2.1-8bit"),
+        name="MiniMax M2.1 8bit",
+        description="MiniMax M2.1 8bit",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/MiniMax-M2.1-8bit"),
+            pretty_name="MiniMax M2.1 8bit",
+            storage_size=Memory.from_bytes(242986745856),
+            n_layers=61,
+            hidden_size=3072,
+            supports_tensor=True,
+        ),
+    ),
+    "minimax-m2.1-3bit": ModelCard(
+        short_id="minimax-m2.1-3bit",
+        model_id=ModelId("mlx-community/MiniMax-M2.1-3bit"),
+        name="MiniMax M2.1 3bit",
+        description="MiniMax M2.1 3bit",
+        tags=[],
+        metadata=ModelMetadata(
+            model_id=ModelId("mlx-community/MiniMax-M2.1-3bit"),
+            pretty_name="MiniMax M2.1 3bit",
+            storage_size=Memory.from_bytes(100086644736),
+            n_layers=61,
+            hidden_size=3072,
+            supports_tensor=True,
+        ),
+    ),
+}

src/exo/shared/models/model_meta.py

@@ -6,6 +6,7 @@ from huggingface_hub import model_info
 from loguru import logger
 from pydantic import BaseModel, Field
 
+from exo.shared.models.model_cards import MODEL_CARDS
 from exo.shared.types.memory import Memory
 from exo.shared.types.models import ModelId, ModelMetadata
 from exo.worker.download.download_utils import (
@@ -107,13 +108,19 @@ async def _get_model_meta(model_id: str) -> ModelMetadata:
     config_data = await get_config_data(model_id)
     num_layers = config_data.layer_count
     mem_size_bytes = await get_safetensors_size(model_id)
+    model_card = next(
+        (card for card in MODEL_CARDS.values() if card.model_id == ModelId(model_id)),
+        None,
+    )
 
     return ModelMetadata(
         model_id=ModelId(model_id),
-        pretty_name=model_id,
+        pretty_name=model_card.name if model_card is not None else model_id,
         storage_size=mem_size_bytes,
         n_layers=num_layers,
         hidden_size=config_data.hidden_size or 0,
         # TODO: all custom models currently do not support tensor. We could add a dynamic test for this?
-        supports_tensor=False,
+        supports_tensor=model_card.metadata.supports_tensor
+        if model_card is not None
+        else False,
     )

src/exo/worker/engines/mlx/utils_mlx.py

@@ -20,6 +20,7 @@ except ImportError:
 
 from mlx_lm.models.cache import KVCache, QuantizedKVCache, RotatingKVCache
 from mlx_lm.models.deepseek_v3 import DeepseekV3Model
+from mlx_lm.models.gpt_oss import Model as GptOssModel
 from mlx_lm.tokenizer_utils import TokenizerWrapper
 
 from exo.worker.engines.mlx.constants import (
@@ -365,6 +366,8 @@ def apply_chat_template(
         tools=chat_task_data.tools,
     )
 
+    logger.info(prompt)
+
     return prompt
 
 
@@ -396,6 +399,11 @@ def make_kv_cache(
 ) -> list[KVCache | RotatingKVCache | QuantizedKVCache]:
     assert hasattr(model, "layers")
 
+    # TODO: Do this for all models
+    if hasattr(model, "make_cache") and isinstance(model, GptOssModel):
+        logger.info("Using MLX LM's make cache")
+        return model.make_cache()  # type: ignore
+
     if max_kv_size is None:
         if KV_CACHE_BITS is None:
             logger.info("Using default KV cache")

src/exo/worker/runner/runner.py

@@ -1,6 +1,15 @@
 import time
+from collections.abc import Generator
+from functools import cache
 
 import mlx.core as mx
+from mlx_lm.models.gpt_oss import Model as GptOssModel
+from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
+    HarmonyEncodingName,
+    Role,
+    StreamableParser,
+    load_harmony_encoding,
+)
 
 from exo.shared.types.api import ChatCompletionMessageText
 from exo.shared.types.chunks import TokenChunk
@@ -153,11 +162,19 @@ def main(
                     _check_for_debug_prompts(task_params.messages[0].content)
 
                     # Generate responses using the actual MLX generation
-                    for response in mlx_generate(
+                    mlx_generator = mlx_generate(
                         model=model,
                         tokenizer=tokenizer,
                         task=task_params,
-                    ):
+                    )
+
+                    # GPT-OSS specific parsing to match other model formats.
+                    if isinstance(model, GptOssModel):
+                        mlx_generator = parse_gpt_oss(mlx_generator)
+
+                    # TODO: Add tool call parser here
+
+                    for response in mlx_generator:
                         match response:
                             case GenerationResponse():
                                 if shard_metadata.device_rank == 0:
@@ -207,6 +224,43 @@ def main(
                 break
 
 
+@cache
+def get_gpt_oss_encoding():
+    encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
+    return encoding
+
+
+def parse_gpt_oss(
+    responses: Generator[GenerationResponse],
+) -> Generator[GenerationResponse]:
+    encoding = get_gpt_oss_encoding()
+    stream = StreamableParser(encoding, role=Role.ASSISTANT)
+    thinking = False
+
+    for response in responses:
+        stream.process(response.token)
+
+        delta = stream.last_content_delta
+        ch = stream.current_channel
+
+        if ch == "analysis" and not thinking:
+            thinking = True
+            yield response.model_copy(update={"text": "<think>"})
+
+        if ch != "analysis" and thinking:
+            thinking = False
+            yield response.model_copy(update={"text": "</think>"})
+
+        if delta:
+            yield response.model_copy(update={"text": delta})
+
+        if response.finish_reason is not None:
+            if thinking:
+                yield response.model_copy(update={"text": "</think>"})
+            yield response
+            break
+
+
 EXO_RUNNER_MUST_FAIL = "EXO RUNNER MUST FAIL"
 EXO_RUNNER_MUST_OOM = "EXO RUNNER MUST OOM"
 EXO_RUNNER_MUST_TIMEOUT = "EXO RUNNER MUST TIMEOUT"

src/exo/worker/utils/net_profile.py

@@ -1,49 +1,64 @@
-import http.client
-
-from anyio import create_task_group, to_thread
+import anyio
+import httpx
+from anyio import create_task_group
 from loguru import logger
 
 from exo.shared.topology import Topology
 from exo.shared.types.common import NodeId
 
+REACHABILITY_ATTEMPTS = 3
+
 
 async def check_reachability(
     target_ip: str,
     expected_node_id: NodeId,
-    self_node_id: NodeId,
     out: dict[NodeId, set[str]],
+    client: httpx.AsyncClient,
 ) -> None:
     """Check if a node is reachable at the given IP and verify its identity."""
+    if ":" in target_ip:
+        # TODO: use real IpAddress types
+        target_ip = f"[{target_ip}]"
+    url = f"http://{target_ip}:52415/node_id"
 
-    def _fetch_remote_node_id() -> NodeId | None:
-        connection = http.client.HTTPConnection(target_ip, 52415, timeout=1)
+    remote_node_id = None
+    last_error = None
+
+    for _ in range(REACHABILITY_ATTEMPTS):
         try:
-            connection.request("GET", "/node_id")
-            response = connection.getresponse()
-            if response.status != 200:
-                return None
+            r = await client.get(url)
+            if r.status_code != 200:
+                await anyio.sleep(1)
+                continue
 
-            body = response.read().decode("utf-8").strip()
+            body = r.text.strip().strip('"')
+            if not body:
+                await anyio.sleep(1)
+                continue
 
-            # Strip quotes if present (JSON string response)
-            if body.startswith('"') and body.endswith('"') and len(body) >= 2:
-                body = body[1:-1]
+            remote_node_id = NodeId(body)
+            break
 
-            return NodeId(body) or None
-        except OSError:
-            return None
-        except http.client.HTTPException:
-            return None
-        finally:
-            connection.close()
+        # expected failure cases
+        except (
+            httpx.TimeoutException,
+            httpx.NetworkError,
+        ):
+            await anyio.sleep(1)
+
+        # other failures should be logged on last attempt
+        except httpx.HTTPError as e:
+            last_error = e
+            await anyio.sleep(1)
+
+    if last_error is not None:
+        logger.warning(
+            f"connect error {type(last_error).__name__} from {target_ip} after {REACHABILITY_ATTEMPTS} attempts; treating as down"
+        )
 
-    remote_node_id = await to_thread.run_sync(_fetch_remote_node_id)
     if remote_node_id is None:
         return
 
-    if remote_node_id == self_node_id:
-        return
-
     if remote_node_id != expected_node_id:
         logger.warning(
             f"Discovered node with unexpected node_id; "
@@ -61,18 +76,33 @@ async def check_reachable(
     topology: Topology, self_node_id: NodeId
 ) -> dict[NodeId, set[str]]:
     """Check which nodes are reachable and return their IPs."""
+
     reachable: dict[NodeId, set[str]] = {}
-    async with create_task_group() as tg:
+
+    # these are intentionally httpx's defaults so we can tune them later
+    timeout = httpx.Timeout(timeout=5.0)
+    limits = httpx.Limits(
+        max_connections=100,
+        max_keepalive_connections=20,
+        keepalive_expiry=5,
+    )
+
+    async with (
+        httpx.AsyncClient(timeout=timeout, limits=limits) as client,
+        create_task_group() as tg,
+    ):
         for node in topology.list_nodes():
             if not node.node_profile:
                 continue
+            if node.node_id == self_node_id:
+                continue
             for iface in node.node_profile.network_interfaces:
                 tg.start_soon(
                     check_reachability,
                     iface.ip_address,
                     node.node_id,
-                    self_node_id,
                     reachable,
+                    client,
                 )
 
     return reachable

uv.lock

@@ -236,6 +236,7 @@ dependencies = [
     { name = "exo-pyo3-bindings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+    { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "hypercorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "loguru", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -247,7 +248,6 @@ dependencies = [
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rustworkx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "tiktoken", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "tomlkit", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "types-aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
 
@@ -269,6 +269,7 @@ requires-dist = [
     { name = "exo-pyo3-bindings", editable = "rust/exo_pyo3_bindings" },
     { name = "fastapi", specifier = ">=0.116.1" },
     { name = "filelock", specifier = ">=3.18.0" },
+    { name = "httpx", specifier = ">=0.28.1" },
     { name = "huggingface-hub", specifier = ">=0.33.4" },
     { name = "hypercorn", specifier = ">=0.18.0" },
     { name = "loguru", specifier = ">=0.7.3" },
@@ -280,7 +281,6 @@ requires-dist = [
     { name = "pydantic", specifier = ">=2.11.7" },
     { name = "rustworkx", specifier = ">=0.17.1" },
     { name = "tiktoken", specifier = ">=0.12.0" },
-    { name = "tomlkit", specifier = ">=0.14.0" },
     { name = "types-aiofiles", specifier = ">=24.1.0.20250708" },
 ]
 
@@ -1380,15 +1380,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" },
 ]
 
-[[package]]
-name = "tomlkit"
-version = "0.14.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" },
-]
-
 [[package]]
 name = "tqdm"
 version = "4.67.1"