wah

## Motivation

Fixes #1918. Downloaded model status reverts from "completed" to
"pending" during each download scan. Reproduced with `zai-org/GLM-5.1`.

## Changes

- `coordinator.py`: In the periodic rescan, don't downgrade
already-completed models; fall back to `resolve_existing_model()`
(safetensors weight check) when per-file size check reports incomplete
- New `test_download_status_not_lost.py`: 3 regression tests

## Why It Works

The rescan compares local file sizes against HF's `main` revision. When
HF updates text files (README, jinja, etc.), remote sizes change but
local files still match the old revision — causing a false "incomplete".
The fix uses the safetensors weight check as ground truth instead.

Long-term: pin the downloaded revision SHA rather than always checking
against `main`.

## Test Plan

### Manual Testing

- Mac Studio M3 Ultra with GLM-5.1 downloaded (natural reproduction of
the issue)
- Confirmed GLM-5.1 stays `DownloadCompleted` through multiple rescan
cycles

### Automated Testing

- 3 new tests: completed-not-downgraded, fallback-to-resolve,
genuinely-incomplete-stays-pending

.github/workflows/build-app.yml

@@ -239,6 +239,80 @@ jobs:
           # Export keychain path for other steps
           echo "BUILD_KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV
 
+      # ============================================================
+      # Pre-flight credential / profile validation
+      # Runs BEFORE the ~16 min build so auth/expiry failures surface in <1 min.
+      # ============================================================
+
+      - name: Validate Apple notarization credentials
+        env:
+          APPLE_NOTARIZATION_USERNAME: ${{ secrets.APPLE_NOTARIZATION_USERNAME }}
+          APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
+          APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
+        run: |
+          # All-or-nothing: either all three creds are set, or none are.
+          CRED_COUNT=0
+          for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
+            [[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
+          done
+          if [[ "$CRED_COUNT" -eq 0 ]]; then
+            echo "No notarization credentials configured — skipping notarization for this build."
+            exit 0
+          fi
+          if [[ "$CRED_COUNT" -ne 3 ]]; then
+            echo "ERROR: partial notarization credentials set ($CRED_COUNT/3). Aborting before build."
+            exit 1
+          fi
+          # Cheap, ~5s, auth-only call. Fails instantly with a clear message if
+          # the app-specific password is stale, wrong team-id, etc.
+          echo "Verifying Apple notarization credentials via notarytool history..."
+          if ! xcrun notarytool history \
+            --apple-id "$APPLE_NOTARIZATION_USERNAME" \
+            --password "$APPLE_NOTARIZATION_PASSWORD" \
+            --team-id "$APPLE_NOTARIZATION_TEAM" >/dev/null; then
+            echo "ERROR: notarytool rejected the provided credentials. Fix before rerunning."
+            echo "Common causes: app-specific password expired/revoked, wrong team-id,"
+            echo "Apple ID not on the team, or 2FA not configured for this Apple ID."
+            exit 1
+          fi
+          echo "Apple notarization credentials OK."
+
+      - name: Validate provisioning profile expiry
+        run: |
+          PROFILE="$HOME/Library/Developer/Xcode/UserData/Provisioning Profiles/EXO.provisionprofile"
+          if [[ ! -f "$PROFILE" ]]; then
+            echo "ERROR: provisioning profile not found at $PROFILE"
+            exit 1
+          fi
+          EXPIRY=$(security cms -D -i "$PROFILE" | plutil -extract ExpirationDate raw -o - - 2>/dev/null || true)
+          if [[ -z "$EXPIRY" ]]; then
+            echo "WARNING: could not read ExpirationDate from provisioning profile; skipping expiry check."
+            exit 0
+          fi
+          # Try a couple of known plutil date formats. If none parse, skip the check rather
+          # than risk a false-positive "expired" block on a format we didn't anticipate.
+          EXPIRY_EPOCH=""
+          for fmt in "%Y-%m-%dT%H:%M:%SZ" "%Y-%m-%d %H:%M:%S %z" "%Y-%m-%d %H:%M:%S +0000"; do
+            if parsed=$(date -j -f "$fmt" "$EXPIRY" +%s 2>/dev/null); then
+              EXPIRY_EPOCH="$parsed"
+              break
+            fi
+          done
+          if [[ -z "$EXPIRY_EPOCH" ]]; then
+            echo "WARNING: could not parse ExpirationDate '$EXPIRY'; skipping expiry check."
+            exit 0
+          fi
+          NOW_EPOCH=$(date +%s)
+          if [[ "$EXPIRY_EPOCH" -le "$NOW_EPOCH" ]]; then
+            echo "ERROR: provisioning profile expired on $EXPIRY. Regenerate it before rerunning."
+            exit 1
+          fi
+          DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
+          echo "Provisioning profile valid until $EXPIRY ($DAYS_LEFT days remaining)."
+          if [[ "$DAYS_LEFT" -lt 14 ]]; then
+            echo "WARNING: profile expires in under 14 days — regenerate soon."
+          fi
+
       # ============================================================
       # Build the bundle
       # ============================================================
@@ -306,11 +380,41 @@ jobs:
           APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
           APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
         run: |
+          set -o pipefail
           cd output
           security unlock-keychain -p "$MACOS_CERTIFICATE_PASSWORD" "$BUILD_KEYCHAIN_PATH"
           SIGNING_IDENTITY=$(security find-identity -v -p codesigning "$BUILD_KEYCHAIN_PATH" | awk -F '"' '{print $2}')
+
+          # Fail fast if notarization creds are partial. All-or-nothing.
+          CRED_COUNT=0
+          for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
+            [[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
+          done
+          if [[ "$CRED_COUNT" -ne 0 && "$CRED_COUNT" -ne 3 ]]; then
+            echo "ERROR: partial Apple notarization credentials set ($CRED_COUNT/3). Aborting."
+            exit 1
+          fi
+
           /usr/bin/codesign --deep --force --timestamp --options runtime \
             --sign "$SIGNING_IDENTITY" EXO.app
+
+          # Pre-flight: verify the signed app BEFORE building DMG and submitting to Apple.
+          # If this fails, notarization will fail too — cheap way to fail in seconds, not 15 minutes.
+          echo "===== codesign --verify EXO.app ====="
+          if ! /usr/bin/codesign --verify --deep --strict --verbose=2 EXO.app; then
+            echo "ERROR: EXO.app failed codesign verification. Dumping signing status of every executable:"
+            find EXO.app -type f \( -perm -111 -o -name "*.dylib" -o -name "*.so" -o -name "*.framework" \) -print0 |
+              while IFS= read -r -d '' f; do
+                printf -- '--- %s\n' "$f"
+                /usr/bin/codesign -dv --verbose=2 "$f" 2>&1 | sed 's/^/    /' || true
+              done
+            exit 1
+          fi
+
+          # Gatekeeper assessment. A failure here strongly predicts notarization rejection.
+          echo "===== spctl assessment (predicts notarization outcome) ====="
+          /usr/bin/spctl -a -vvv -t install EXO.app || echo "WARNING: spctl assessment failed — notarization is likely to fail too."
+
           mkdir -p dmg-root
           cp -R EXO.app dmg-root/
           ln -s /Applications dmg-root/Applications
@@ -318,12 +422,22 @@ jobs:
           hdiutil create -volname "EXO" -srcfolder dmg-root -ov -format UDZO "$DMG_NAME"
           /usr/bin/codesign --force --timestamp --options runtime \
             --sign "$SIGNING_IDENTITY" "$DMG_NAME"
+
+          echo "===== codesign --verify DMG ====="
+          if ! /usr/bin/codesign --verify --verbose=2 "$DMG_NAME"; then
+            echo "ERROR: DMG failed codesign verification."
+            exit 1
+          fi
+
           if [[ -n "$APPLE_NOTARIZATION_USERNAME" ]]; then
+            echo "===== notarytool submit ====="
+            # `|| true` so set -e doesn't abort before we can echo output / fetch the log.
+            # We rely on the parsed STATUS below to decide pass/fail.
             SUBMISSION_OUTPUT=$(xcrun notarytool submit "$DMG_NAME" \
               --apple-id "$APPLE_NOTARIZATION_USERNAME" \
               --password "$APPLE_NOTARIZATION_PASSWORD" \
               --team-id "$APPLE_NOTARIZATION_TEAM" \
-              --wait --timeout 15m 2>&1)
+              --wait --timeout 15m 2>&1) || true
             echo "$SUBMISSION_OUTPUT"
 
             SUBMISSION_ID=$(echo "$SUBMISSION_OUTPUT" | awk 'tolower($1)=="id:" && $2 ~ /^[0-9a-fA-F-]+$/ {print $2; exit}')

.github/workflows/pipeline.yml

@@ -91,9 +91,6 @@ jobs:
             nix build .#metal-toolchain
           fi
 
-          # Build mlx (depends on metal-toolchain)
-          nix build .#mlx
-
       - name: Build all Nix outputs
         run: |
           nix flake show --json | jq -r '

.mlx_typings/mlx_lm/sample_utils.pyi

@@ -48,6 +48,10 @@ def make_logits_processors(
     logit_bias: Optional[Dict[int, float]] = ...,
     repetition_penalty: Optional[float] = ...,
     repetition_context_size: Optional[int] = ...,
+    presence_penalty: Optional[float] = ...,
+    presence_context_size: Optional[int] = ...,
+    frequency_penalty: Optional[float] = ...,
+    frequency_context_size: Optional[int] = ...,
 ) -> list[Callable[[mx.array, mx.array], mx.array]]:
     """
     Make logits processors for use with ``generate_step``.

app/EXO/EXO/ExoProcessController.swift

@@ -9,11 +9,14 @@ private let enableImageModelsKey = "EXOEnableImageModels"
 private let offlineModeKey = "EXOOfflineMode"
 private let fastSynchEnabledKey = "EXOFastSynchEnabled"
 private let onboardingCompletedKey = "EXOOnboardingCompleted"
+private let defaultModelsDirKey = "EXODefaultModelsDir"
+private let additionalModelsDirsKey = "EXOAdditionalModelsDirs"
+private let readOnlyModelsDirsKey = "EXOReadOnlyModelsDirs"
 private let customEnvironmentVariablesKey = "EXOCustomEnvironmentVariables"
 
 /// A user-defined environment variable that is injected into the exo child
-/// process at launch. Used to pass arbitrary key/value settings to exo
-/// without having to add first-class UI for each one.
+/// process at launch. Used as an escape hatch for env vars that don't have
+/// first-class typed UI in Settings.
 struct CustomEnvironmentVariable: Codable, Identifiable, Equatable {
     var id: UUID
     var key: String
@@ -106,6 +109,30 @@ final class ExoProcessController: ObservableObject {
             UserDefaults.standard.set(fastSynchEnabled, forKey: fastSynchEnabledKey)
         }
     }
+    @Published var defaultModelsDir: String = {
+        return UserDefaults.standard.string(forKey: defaultModelsDirKey) ?? ""
+    }()
+    {
+        didSet {
+            UserDefaults.standard.set(defaultModelsDir, forKey: defaultModelsDirKey)
+        }
+    }
+    @Published var additionalModelsDirs: String = {
+        return UserDefaults.standard.string(forKey: additionalModelsDirsKey) ?? ""
+    }()
+    {
+        didSet {
+            UserDefaults.standard.set(additionalModelsDirs, forKey: additionalModelsDirsKey)
+        }
+    }
+    @Published var readOnlyModelsDirs: String = {
+        return UserDefaults.standard.string(forKey: readOnlyModelsDirsKey) ?? ""
+    }()
+    {
+        didSet {
+            UserDefaults.standard.set(readOnlyModelsDirs, forKey: readOnlyModelsDirsKey)
+        }
+    }
     @Published var customEnvironmentVariables: [CustomEnvironmentVariable] = {
         guard
             let data = UserDefaults.standard.data(forKey: customEnvironmentVariablesKey),
@@ -364,8 +391,21 @@ final class ExoProcessController: ObservableObject {
 
         environment["PATH"] = paths.joined(separator: ":")
 
+        let trimmedDefaultModelsDir = defaultModelsDir.trimmingCharacters(in: .whitespaces)
+        if !trimmedDefaultModelsDir.isEmpty {
+            environment["EXO_DEFAULT_MODELS_DIR"] = trimmedDefaultModelsDir
+        }
+        let trimmedAdditionalModelsDirs = additionalModelsDirs.trimmingCharacters(in: .whitespaces)
+        if !trimmedAdditionalModelsDirs.isEmpty {
+            environment["EXO_MODELS_DIRS"] = trimmedAdditionalModelsDirs
+        }
+        let trimmedReadOnlyModelsDirs = readOnlyModelsDirs.trimmingCharacters(in: .whitespaces)
+        if !trimmedReadOnlyModelsDirs.isEmpty {
+            environment["EXO_MODELS_READ_ONLY_DIRS"] = trimmedReadOnlyModelsDirs
+        }
+
         // Apply user-defined arbitrary environment variables last so that
-        // power users can override any of the built-in keys above when
+        // power users can override any of the typed fields above when
         // necessary. Empty keys are ignored.
         for variable in customEnvironmentVariables {
             let trimmedKey = variable.key.trimmingCharacters(in: .whitespaces)

app/EXO/EXO/Views/SettingsView.swift

@@ -16,6 +16,9 @@ struct SettingsView: View {
     @State private var pendingEnableImageModels = false
     @State private var pendingOfflineMode = false
     @State private var pendingFastSynchEnabled = false
+    @State private var pendingDefaultModelsDir: String = ""
+    @State private var pendingAdditionalModelsDirs: String = ""
+    @State private var pendingReadOnlyModelsDirs: String = ""
     @State private var pendingCustomEnvironmentVariables: [CustomEnvironmentVariable] = []
     @State private var needsRestart = false
     @State private var bugReportInFlight = false
@@ -45,7 +48,7 @@ struct SettingsView: View {
                     Label("About", systemImage: "info.circle")
                 }
         }
-        .frame(width: 450, height: 400)
+        .frame(width: 640, height: 560)
         .onAppear {
             pendingNamespace = controller.customNamespace
             pendingHFToken = controller.hfToken
@@ -53,6 +56,9 @@ struct SettingsView: View {
             pendingEnableImageModels = controller.enableImageModels
             pendingOfflineMode = controller.offlineMode
             pendingFastSynchEnabled = controller.fastSynchEnabled
+            pendingDefaultModelsDir = controller.defaultModelsDir
+            pendingAdditionalModelsDirs = controller.additionalModelsDirs
+            pendingReadOnlyModelsDirs = controller.readOnlyModelsDirs
             pendingCustomEnvironmentVariables = controller.customEnvironmentVariables
             needsRestart = false
         }
@@ -64,9 +70,9 @@ struct SettingsView: View {
         Form {
             Section {
                 LabeledContent("Cluster Namespace") {
-                    TextField("default", text: $pendingNamespace)
+                    TextField("", text: $pendingNamespace, prompt: Text("default"))
                         .textFieldStyle(.roundedBorder)
-                        .frame(width: 200)
+                        .frame(width: 260)
                 }
                 Text("Nodes with the same namespace form a cluster. Leave empty for default.")
                     .font(.caption)
@@ -75,9 +81,9 @@ struct SettingsView: View {
 
             Section {
                 LabeledContent("HuggingFace Token") {
-                    SecureField("optional", text: $pendingHFToken)
+                    SecureField("", text: $pendingHFToken, prompt: Text("optional"))
                         .textFieldStyle(.roundedBorder)
-                        .frame(width: 200)
+                        .frame(width: 260)
                 }
                 Text("Required for gated models. Get yours at huggingface.co/settings/tokens")
                     .font(.caption)
@@ -86,9 +92,9 @@ struct SettingsView: View {
 
             Section {
                 LabeledContent("HuggingFace Endpoint") {
-                    TextField("default", text: $pendingHFEndpoint)
+                    TextField("", text: $pendingHFEndpoint, prompt: Text("default"))
                         .textFieldStyle(.roundedBorder)
-                        .frame(width: 200)
+                        .frame(width: 260)
                 }
                 Text("Defaults to huggingface.co. Use a mirror (e.g. hf-mirror.com) for China.")
                     .font(.caption)
@@ -222,11 +228,58 @@ struct SettingsView: View {
 
     private var environmentTab: some View {
         Form {
-            Section("Custom Environment Variables") {
-                Text("Passed to the exo process at launch. Override built-in defaults here.")
+            Section("Models Directories") {
+                LabeledContent("Default Models Directory") {
+                    TextField(
+                        "",
+                        text: $pendingDefaultModelsDir,
+                        prompt: Text("~/.exo/models")
+                    )
+                    .textFieldStyle(.roundedBorder)
+                    .font(.system(.body, design: .monospaced))
+                    .frame(width: 260)
+                }
+                Text("Sets EXO_DEFAULT_MODELS_DIR. Where models are downloaded.")
                     .font(.caption)
                     .foregroundColor(.secondary)
 
+                LabeledContent("Additional Directories") {
+                    TextField(
+                        "",
+                        text: $pendingAdditionalModelsDirs,
+                        prompt: Text("optional, colon-separated")
+                    )
+                    .textFieldStyle(.roundedBorder)
+                    .font(.system(.body, design: .monospaced))
+                    .frame(width: 260)
+                }
+                Text("Sets EXO_MODELS_DIRS. Extra writable model directories.")
+                    .font(.caption)
+                    .foregroundColor(.secondary)
+
+                LabeledContent("Read-Only Directories") {
+                    TextField(
+                        "",
+                        text: $pendingReadOnlyModelsDirs,
+                        prompt: Text("optional, colon-separated")
+                    )
+                    .textFieldStyle(.roundedBorder)
+                    .font(.system(.body, design: .monospaced))
+                    .frame(width: 260)
+                }
+                Text("Sets EXO_MODELS_READ_ONLY_DIRS. Never written to.")
+                    .font(.caption)
+                    .foregroundColor(.secondary)
+            }
+
+            Section("Custom Environment Variables") {
+                Text(
+                    "Escape hatch for env vars that don't have typed fields above. "
+                        + "Values here override the typed fields on conflict."
+                )
+                .font(.caption)
+                .foregroundColor(.secondary)
+
                 if pendingCustomEnvironmentVariables.isEmpty {
                     Text("No custom variables.")
                         .font(.caption)
@@ -580,7 +633,10 @@ struct SettingsView: View {
     }
 
     private var hasEnvironmentChanges: Bool {
-        pendingCustomEnvironmentVariables != controller.customEnvironmentVariables
+        pendingDefaultModelsDir != controller.defaultModelsDir
+            || pendingAdditionalModelsDirs != controller.additionalModelsDirs
+            || pendingReadOnlyModelsDirs != controller.readOnlyModelsDirs
+            || pendingCustomEnvironmentVariables != controller.customEnvironmentVariables
     }
 
     private func applyGeneralSettings() {
@@ -602,6 +658,17 @@ struct SettingsView: View {
     }
 
     private func applyEnvironmentSettings() {
+        controller.defaultModelsDir = pendingDefaultModelsDir.trimmingCharacters(
+            in: .whitespaces)
+        controller.additionalModelsDirs = pendingAdditionalModelsDirs.trimmingCharacters(
+            in: .whitespaces)
+        controller.readOnlyModelsDirs = pendingReadOnlyModelsDirs.trimmingCharacters(
+            in: .whitespaces)
+
+        pendingDefaultModelsDir = controller.defaultModelsDir
+        pendingAdditionalModelsDirs = controller.additionalModelsDirs
+        pendingReadOnlyModelsDirs = controller.readOnlyModelsDirs
+
         // Trim whitespace from keys and drop empty ones so that the stored
         // form matches what is actually injected into the child process and
         // hasEnvironmentChanges doesn't show a stale diff after save.
@@ -629,6 +696,7 @@ struct SettingsView: View {
 
         pendingCustomEnvironmentVariables = sanitized
         controller.customEnvironmentVariables = sanitized
+
         restartIfRunning()
     }
 

app/EXO/EXO/Views/SettingsWindowController.swift

@@ -30,7 +30,7 @@ final class SettingsWindowController: ObservableObject {
         let hostingView = NSHostingView(rootView: settingsView)
 
         let newWindow = NSWindow(
-            contentRect: NSRect(x: 0, y: 0, width: 450, height: 400),
+            contentRect: NSRect(x: 0, y: 0, width: 640, height: 560),
             styleMask: [.titled, .closable],
             backing: .buffered,
             defer: false

bench/METHODOLOGY.md

@@ -28,7 +28,7 @@ Chat template formatting means that it may be impossible to attain very small pp
 
 When a request reaches the server via the `/bench/chat/completions` endpoint, three things change compared to a normal chat completion:
 
-- **KV prefix cache is disabled**. Every request starts from a cold cache, ensuring prefill timing is not affected by prior requests.
+- **KV prefix cache is disabled by default**. Every request starts from a cold cache, ensuring prefill timing is not affected by prior requests. See [Prefix Cache Mode](#prefix-cache-mode) for the `--use-prefix-cache` option.
 - **EOS tokens are banned**. A logits processor suppresses all end-of-sequence tokens, forcing the model to generate exactly `max_tokens` tokens. This guarantees consistent generation length for fair TPS comparison — the model cannot short-circuit a run by stopping early.
 - **No model output parsing**. The bench collection path concatenates raw token text without any model-specific post-processing (thinking tag extraction, structured output handling, etc.). This is to avoid model outputs such as tool parsing or any structural mistakes from breaking the benchmark - we are testing for speed; see Exo-Eval for performance metrics.
 
@@ -94,6 +94,22 @@ agg_gen_tps = per_req_tps * concurrency
 
 ---
 
+## Prefix Cache Mode
+
+When `--use-prefix-cache` is passed, the KV prefix cache remains active during benchmarking. This speeds up repeated runs by skipping redundant prefill work, which is useful when prompt processing is not the focus of the benchmark (e.g. when measuring generation throughput or power consumption across many configurations).
+
+Each response includes a `prefix_cache_hit` field (`"none"`, `"partial"`, or `"exact"`):
+
+- **none**: Cold prefill — no cached KV state was available. The reported `prompt_tps` is the real prefill throughput.
+- **partial**: A prefix of the prompt was found in cache. Only the remaining tokens were prefilled. The reported `prompt_tps` reflects the real throughput on the uncached portion. This occurs when multiple ascending `--pp` values share a common prefix (e.g. `--pp 1000,5000` — the 5000-token prompt reuses the 1000-token cache entry and prefills the remaining 4000 tokens).
+- **exact**: The entire prompt was found in cache (e.g. same `--pp` value on a `--repeat`). No prefill work was done. The reported `prompt_tps` is the TPS from when the cache entry was originally created, not a new measurement.
+
+**Prompt TPS is approximate in this mode.** Exact-hit runs report the stored TPS from the original cold/partial prefill rather than a freshly measured value. For accurate cold prefill numbers, run without `--use-prefix-cache`.
+
+Ascending `--pp` order (e.g. `--pp 1000,5000,10000`) gives the most useful data: each size gets a meaningful partial hit except the first which is cold. Descending order produces exact hits with approximate TPS from a longer prompt's original run.
+
+---
+
 ## Warmup
 
 Before measurement begins, `--warmup N` (default: 0) discarded requests are sent using the first pp/tg pair. Warmup results are not included in the output.

bench/exo_bench.py

@@ -230,7 +230,13 @@ def parse_int_list(values: list[str]) -> list[int]:
 
 
 def run_one_completion(
-    client: ExoClient, model_id: str, pp_hint: int, tg: int, prompt_sizer: PromptSizer
+    client: ExoClient,
+    model_id: str,
+    pp_hint: int,
+    tg: int,
+    prompt_sizer: PromptSizer,
+    *,
+    use_prefix_cache: bool = False,
 ) -> tuple[dict[str, Any], int]:
     content, pp_tokens = prompt_sizer.build(pp_hint)
     payload: dict[str, Any] = {
@@ -239,6 +245,7 @@ def run_one_completion(
         "stream": False,
         "max_tokens": tg,
         "logprobs": False,
+        "use_prefix_cache": use_prefix_cache,
     }
 
     t0 = time.perf_counter()
@@ -379,6 +386,11 @@ def main() -> int:
         default=1.0,
         help="System metrics polling interval in seconds (default: 1.0).",
     )
+    ap.add_argument(
+        "--use-prefix-cache",
+        action="store_true",
+        help="Enable KV prefix cache during bench (default: disabled for cold-cache measurements).",
+    )
     args = ap.parse_args()
 
     pp_list = parse_int_list(args.pp)
@@ -394,6 +406,15 @@ def main() -> int:
         logger.error("--concurrency values must be >= 1")
         return 2
 
+    if args.use_prefix_cache:
+        logger.warning(
+            "--use-prefix-cache: prompt TPS will be approximate. See METHODOLOGY.md for details."
+        )
+        if pp_list != sorted(pp_list):
+            logger.warning(
+                "--pp values are not in ascending order: prompt TPS will be less accurate. Use ascending --pp for best results."
+            )
+
     # Log pairing mode
     use_combinations = args.all_combinations or len(pp_list) != len(tg_list)
     if use_combinations:
@@ -505,7 +526,12 @@ def main() -> int:
         try:
             for i in range(args.warmup):
                 run_one_completion(
-                    client, full_model_id, pp_list[0], tg_list[0], prompt_sizer
+                    client,
+                    full_model_id,
+                    pp_list[0],
+                    tg_list[0],
+                    prompt_sizer,
+                    use_prefix_cache=args.use_prefix_cache,
                 )
                 logger.debug(f"  warmup {i + 1}/{args.warmup} done")
 
@@ -529,7 +555,12 @@ def main() -> int:
                             try:
                                 inf_t0 = time.monotonic()
                                 row, actual_pp_tokens = run_one_completion(
-                                    client, full_model_id, pp, tg, prompt_sizer
+                                    client,
+                                    full_model_id,
+                                    pp,
+                                    tg,
+                                    prompt_sizer,
+                                    use_prefix_cache=args.use_prefix_cache,
                                 )
                                 inference_windows.append((inf_t0, time.monotonic()))
                             except Exception as e:
@@ -566,6 +597,7 @@ def main() -> int:
                                 "stream": False,
                                 "max_tokens": tg,
                                 "logprobs": False,
+                                "use_prefix_cache": args.use_prefix_cache,
                             }
                             barrier = threading.Barrier(concurrency)
                             batch_start = threading.Event()

dashboard/src/lib/components/IntegrationCard.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+  import { copyText } from "$lib/utils/clipboard";
+
   interface Props {
     title: string;
     subtitle: string;
@@ -16,11 +18,17 @@
   }: Props = $props();
 
   let copied = $state(false);
+  let failed = $state(false);
 
   async function copyToClipboard() {
-    await navigator.clipboard.writeText(config);
-    copied = true;
-    setTimeout(() => (copied = false), 2000);
+    const ok = await copyText(config);
+    if (ok) {
+      copied = true;
+      setTimeout(() => (copied = false), 2000);
+    } else {
+      failed = true;
+      setTimeout(() => (failed = false), 2000);
+    }
   }
 </script>
 
@@ -37,9 +45,11 @@
       class="px-3 py-1.5 text-xs rounded border transition-all duration-200 cursor-pointer
         {copied
         ? 'border-green-500/50 text-green-400 bg-green-500/10'
-        : 'border-exo-light-gray/30 text-exo-light-gray hover:border-exo-yellow/50 hover:text-exo-yellow'}"
+        : failed
+          ? 'border-red-500/50 text-red-400 bg-red-500/10'
+          : 'border-exo-light-gray/30 text-exo-light-gray hover:border-exo-yellow/50 hover:text-exo-yellow'}"
     >
-      {copied ? "Copied!" : "Copy"}
+      {copied ? "Copied!" : failed ? "Copy failed" : "Copy"}
     </button>
   </div>
   {#if description}

dashboard/src/lib/utils/clipboard.ts

@@ -0,0 +1,55 @@
+export async function copyText(text: string): Promise<boolean> {
+  if (
+    typeof window !== "undefined" &&
+    window.isSecureContext &&
+    navigator.clipboard?.writeText
+  ) {
+    try {
+      await navigator.clipboard.writeText(text);
+      return true;
+    } catch {
+      // fall through to execCommand fallback
+    }
+  }
+
+  if (typeof document === "undefined") {
+    return false;
+  }
+
+  const textarea = document.createElement("textarea");
+  textarea.value = text;
+  textarea.setAttribute("readonly", "");
+  textarea.style.position = "fixed";
+  textarea.style.top = "0";
+  textarea.style.left = "0";
+  textarea.style.width = "1px";
+  textarea.style.height = "1px";
+  textarea.style.padding = "0";
+  textarea.style.border = "none";
+  textarea.style.outline = "none";
+  textarea.style.boxShadow = "none";
+  textarea.style.background = "transparent";
+  textarea.style.opacity = "0";
+  document.body.appendChild(textarea);
+
+  const previousSelection = document.getSelection();
+  const previousRange =
+    previousSelection && previousSelection.rangeCount > 0
+      ? previousSelection.getRangeAt(0)
+      : null;
+
+  try {
+    textarea.focus();
+    textarea.select();
+    textarea.setSelectionRange(0, text.length);
+    return document.execCommand("copy");
+  } catch {
+    return false;
+  } finally {
+    document.body.removeChild(textarea);
+    if (previousRange && previousSelection) {
+      previousSelection.removeAllRanges();
+      previousSelection.addRange(previousRange);
+    }
+  }
+}

docs/architecture.md

@@ -81,4 +81,4 @@ Whenever a device produces side effects, it captures those side effects in an `E
 
 ## Purity
 
-A significant goal of the current design is to make data flow explicit. Classes should either represent simple data (`CamelCaseModel`s typically, and `TaggedModel`s for unions) or active `System`s (Erlang `Actor`s), with all transformations of that data being "referentially transparent" - destructure and construct new data, don't mutate in place. We have had varying degrees of success with this, and are still exploring where purity makes sense.
+A significant goal of the current design is to make data flow explicit. Classes should either represent simple data (`FrozenModel`s typically, and `TaggedModel`s for unions) or active `System`s (Erlang `Actor`s), with all transformations of that data being "referentially transparent" - destructure and construct new data, don't mutate in place. We have had varying degrees of success with this, and are still exploring where purity makes sense.

flake.lock

@@ -96,6 +96,26 @@
         "type": "github"
       }
     },
+    "nixglhost": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732211616,
+        "narHash": "sha256-QZCKJoypcwgS3tDNSWMjlxEBZtOYPW3eXV24rMzKsac=",
+        "owner": "numtide",
+        "repo": "nix-gl-host",
+        "rev": "5269b233f83880a0b433eafe026f0bc0d8f1a4a9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-gl-host",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1775595990,
@@ -187,6 +207,7 @@
         "dream2nix": "dream2nix",
         "fenix": "fenix",
         "flake-parts": "flake-parts",
+        "nixglhost": "nixglhost",
         "nixpkgs": "nixpkgs",
         "pyproject-build-systems": "pyproject-build-systems",
         "pyproject-nix": "pyproject-nix",

flake.nix

@@ -45,11 +45,16 @@
       inputs.uv2nix.follows = "uv2nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    nixglhost = {
+      url = "github:numtide/nix-gl-host";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   nixConfig = {
-    extra-trusted-public-keys = "exo.cachix.org-1:okq7hl624TBeAR3kV+g39dUFSiaZgLRkLsFBCuJ2NZI=";
-    extra-substituters = "https://exo.cachix.org";
+    extra-trusted-public-keys = "exo.cachix.org-1:okq7hl624TBeAR3kV+g39dUFSiaZgLRkLsFBCuJ2NZI= cache.nixos-cuda.org:74DUi4Ye579gUqzH4ziL9IyiJBlDpMRn9MBN8oNan9M=";
+    extra-substituters = "https://exo.cachix.org https://cache.nixos-cuda.org";
   };
 
   outputs = inputs:
@@ -70,12 +75,12 @@
       debug = true; # Enable options autocompletion
 
       perSystem = { config, self', pkgs, lib, system, ... }:
-        {
-          # Allow unfree for metal-toolchain (needed for Darwin Metal packages)
-          _module.args.pkgs = import inputs.nixpkgs {
+        let
+          pkgsArgs = {
             inherit system;
             config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
             overlays = [
+              inputs.nixglhost.overlays.default
               (import ./nix/apple-sdk-overlay.nix)
               (final: _: {
                 macmon = final.rustPlatform.buildRustPackage {
@@ -92,6 +97,13 @@
               })
             ];
           };
+        in
+        {
+          # Allow unfree for metal-toolchain (needed for Darwin Metal packages)
+          _module.args = {
+            pkgs = import inputs.nixpkgs pkgsArgs;
+            unfreePkgs = import inputs.nixpkgs (pkgsArgs // { config.allowUnfree = true; });
+          };
           treefmt = {
             projectRootFile = "flake.nix";
             programs = {
@@ -118,22 +130,12 @@
             };
           };
 
-          packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin (
-            let
-              uvLock = builtins.fromTOML (builtins.readFile ./uv.lock);
-              mlxPackage = builtins.head (builtins.filter (p: p.name == "mlx" && p.source ? git) uvLock.package);
-              uvLockMlxVersion = mlxPackage.version;
-              uvLockMlxRev = builtins.elemAt (builtins.split "#" mlxPackage.source.git) 2;
-            in
-            {
-              metal-toolchain = pkgs.callPackage ./nix/metal-toolchain.nix { };
-              mlx = pkgs.callPackage ./nix/mlx.nix {
-                inherit (self'.packages) metal-toolchain;
-                inherit uvLockMlxVersion uvLockMlxRev;
-              };
-              default = self'.packages.exo;
-            }
-          );
+          packages = {
+            default = self'.packages.exo;
+          } //
+          lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin {
+            metal-toolchain = pkgs.callPackage ./nix/metal-toolchain.nix { };
+          };
 
           devShells.default = with pkgs; pkgs.mkShell {
             inputsFrom = [ self'.checks.cargo-build ];
@@ -144,10 +146,8 @@
                 config.treefmt.build.wrapper
 
                 # PYTHON
-                self'.packages.python
+                self'.packages.editableVenv
                 uv
-                ruff
-                basedpyright
 
                 # RUST
                 config.rust.toolchain
@@ -164,9 +164,6 @@
                 just
                 jq
               ]
-              ++ lib.optionals stdenv.isLinux [
-                unixtools.ifconfig
-              ]
               ++ lib.optionals stdenv.isDarwin [
                 macmon
               ];
@@ -174,7 +171,7 @@
             OPENSSL_NO_VENDOR = "1";
 
             shellHook = ''
-              export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${self'.packages.python}/lib"
+              export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${python313}/lib"
               ${lib.optionalString stdenv.isLinux ''
                 export LD_LIBRARY_PATH="${openssl.out}/lib:$LD_LIBRARY_PATH"
               ''}

justfile

@@ -22,7 +22,7 @@ sync-clean:
     uv sync --all-packages --force-reinstall --no-cache
 
 rust-rebuild:
-    cargo run --bin stub_gen
+    PYO3_PYTHON="$(uv run python -c 'import sys; print(sys.executable)')" cargo run --bin stub_gen
     uv sync --reinstall-package exo_pyo3_bindings
 
 build-dashboard:
@@ -36,7 +36,7 @@ package: build-dashboard
     uv run pyinstaller packaging/pyinstaller/exo.spec
     rm -rf build
 
-build-app: package
+build-app: rust-rebuild sync-clean package
     xcodebuild build -project app/EXO/EXO.xcodeproj -scheme EXO -configuration Debug -derivedDataPath app/EXO/build
     @echo "\nBuild complete. Run with:\n  open {{justfile_directory()}}/app/EXO/build/Build/Products/Debug/EXO.app"
 

nix/mlx.nix

@@ -1,158 +0,0 @@
-{ stdenv
-, lib
-, fetchFromGitHub
-, replaceVars
-, fetchzip
-, cmake
-, nlohmann_json
-, apple-sdk_26
-, metal-toolchain
-, runCommand
-, fmt
-, python313Packages
-, uvLockMlxVersion
-, uvLockMlxRev
-}:
-
-assert stdenv.isDarwin;
-
-let
-  python = python313Packages.python;
-
-  # Static dependencies included directly during compilation
-  gguf-tools = fetchFromGitHub {
-    owner = "antirez";
-    repo = "gguf-tools";
-    rev = "8fa6eb65236618e28fd7710a0fba565f7faa1848";
-    hash = "sha256-15FvyPOFqTOr5vdWQoPnZz+mYH919++EtghjozDlnSA=";
-  };
-
-  metal_cpp = fetchzip {
-    url = "https://developer.apple.com/metal/cpp/files/metal-cpp_26.zip";
-    hash = "sha256-7n2eI2lw/S+Us6l7YPAATKwcIbRRpaQ8VmES7S8ZjY8=";
-  };
-
-  nanobind = fetchFromGitHub {
-    owner = "wjakob";
-    repo = "nanobind";
-    rev = "v2.10.2";
-    hash = "sha256-io44YhN+VpfHFWyvvLWSanRgbzA0whK8WlDNRi3hahU=";
-    fetchSubmodules = true;
-  };
-
-  mlx = stdenv.mkDerivation rec {
-    pname = "mlx";
-    version = uvLockMlxVersion;
-    pyproject = true;
-
-    src = fetchFromGitHub {
-      owner = "rltakashige";
-      repo = "mlx-jaccl-fix-small-recv";
-      rev = uvLockMlxRev;
-      hash = "sha256-M9x9QBYxwHv2z47qGZNJ4FgJyqLSIZ/3G1fEFQ421Lo=";
-    };
-
-    patches = [
-      (replaceVars ./darwin-build-fixes.patch {
-        sdkVersion = apple-sdk_26.version;
-        metalVersion = metal-toolchain.metalVersion;
-      })
-    ];
-
-    postPatch = ''
-      substituteInPlace mlx/backend/cpu/jit_compiler.cpp \
-        --replace-fail "g++" "$CXX"
-    '';
-
-    dontUseCmakeConfigure = true;
-
-    enableParallelBuilding = true;
-
-    # Allows multiple cores to be used in Python builds.
-    postUnpack = ''
-      export MAKEFLAGS+="''${enableParallelBuilding:+-j$NIX_BUILD_CORES}"
-    '';
-
-    # Updates the wrong fetcher rev attribute
-    passthru.skipBulkUpdate = true;
-
-    env = {
-      DEV_RELEASE = 1;
-      CMAKE_ARGS = toString [
-        (lib.cmakeBool "USE_SYSTEM_FMT" true)
-        (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_GGUFLIB" "${gguf-tools}")
-        (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_JSON" "${nlohmann_json.src}")
-        (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_NANOBIND" "${nanobind}")
-        (lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true)
-        (lib.cmakeBool "MLX_BUILD_CPU" true)
-        (lib.cmakeBool "MLX_BUILD_METAL" true)
-        (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_METAL_CPP" "${metal_cpp}")
-        (lib.cmakeOptionType "string" "CMAKE_OSX_DEPLOYMENT_TARGET" "${apple-sdk_26.version}")
-        (lib.cmakeOptionType "filepath" "CMAKE_OSX_SYSROOT" "${apple-sdk_26.passthru.sdkroot}")
-      ];
-      SDKROOT = apple-sdk_26.passthru.sdkroot;
-      MACOSX_DEPLOYMENT_TARGET = apple-sdk_26.version;
-    };
-
-    build-system = [
-      python313Packages.setuptools
-    ];
-
-    nativeBuildInputs = [
-      cmake
-      metal-toolchain
-      python313Packages.pypaBuildHook
-      python313Packages.pypaInstallHook
-      python313Packages.setuptools
-      python313Packages.typing-extensions
-      python313Packages.wheel
-      python313Packages.cmake
-      python313Packages.ninja
-    ];
-
-    buildInputs = [
-      fmt
-      gguf-tools
-      python313Packages.nanobind
-      python313Packages.pybind11
-      apple-sdk_26
-    ];
-
-    # Tests require Metal GPU access which isn't available in the Nix sandbox.
-    # To run tests, build with: nix build --option sandbox false .#mlx.passthru.tests.mlxTest
-    doCheck = false;
-
-    pythonImportsCheck = [ "mlx" ];
-
-    passthru.tests = {
-      # Runs example scripts to verify MLX works. Requires --option sandbox false
-      # since Metal GPU access is needed.
-      mlxTest =
-        runCommand "run-mlx-examples"
-          {
-            buildInputs = [ mlx ];
-            nativeBuildInputs = [ python ];
-          }
-          ''
-            cp ${src}/examples/python/logistic_regression.py .
-            ${python.interpreter} logistic_regression.py
-            rm logistic_regression.py
-
-            cp ${src}/examples/python/linear_regression.py .
-            ${python.interpreter} linear_regression.py
-            rm linear_regression.py
-
-            touch $out
-          '';
-    };
-
-    meta = {
-      homepage = "https://github.com/ml-explore/mlx";
-      description = "Array framework for Apple silicon";
-      changelog = "https://github.com/ml-explore/mlx/releases/tag/${src.tag}";
-      license = lib.licenses.mit;
-      platforms = [ "aarch64-darwin" ];
-    };
-  };
-in
-mlx

packaging/pyinstaller/exo.spec

@@ -1,5 +1,6 @@
 # -*- mode: python ; coding: utf-8 -*-
 
+import sys
 import importlib.util
 import shutil
 from pathlib import Path
@@ -56,6 +57,7 @@ HIDDEN_IMPORTS = sorted(
     set(
         collect_submodules("mlx")
         + _safe_collect("mlx_lm")
+        + _safe_collect("mlx_vlm")
         + _safe_collect("transformers")
     )
 )
@@ -67,18 +69,19 @@ DATAS: list[tuple[str, str]] = [
     (str(EXO_SHARED_MODELS_DIR), "exo/shared/models"),
 ]
 
-MACMON_PATH = shutil.which("macmon")
-if MACMON_PATH is None:
-    raise SystemExit(
-        "macmon binary not found in PATH. "
-        "Install the pinned fork used by exo via: "
-        "cargo install --git https://github.com/vladkens/macmon "
-        "--rev a1cd06b6cc0d5e61db24fd8832e74cd992097a7d macmon --force"
-    )
+if sys.platform == "darwin":
+    MACMON_PATH = shutil.which("macmon")
+    if MACMON_PATH is None:
+        raise SystemExit(
+            "macmon binary not found in PATH. "
+            "Install the pinned fork used by exo via: "
+            "cargo install --git https://github.com/vladkens/macmon "
+            "--rev a1cd06b6cc0d5e61db24fd8832e74cd992097a7d macmon --force"
+        )
 
 BINARIES: list[tuple[str, str]] = [
     (MACMON_PATH, "."),
-]
+] if sys.platform == "darwin" else []
 
 a = Analysis(
     [str(ENTRYPOINT)],

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "exo"
-version = "0.3.69"
+version = "0.3.70"
 description = "Exo"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -15,22 +15,22 @@ dependencies = [
   "huggingface-hub>=1.8.0",
   "psutil>=7.0.0",
   "loguru>=0.7.3",
-  "exo_pyo3_bindings",                         # rust bindings
+  "exo-pyo3-bindings",                       # rust bindings
   "anyio==4.11.0",
-  "mlx; sys_platform == 'darwin'",
-  "mlx[cpu]==0.30.6; sys_platform == 'linux'",
+  "mlx==0.31.1; sys_platform == 'darwin'",
   "mlx-lm",
-  "tiktoken>=0.12.0",                          # required for kimi k2 tokenizer
+  "tiktoken>=0.12.0",                        # required for kimi k2 tokenizer
   "hypercorn>=0.18.0",
   "openai-harmony>=0.0.8",
   "httpx>=0.28.1",
   "tomlkit>=0.14.0",
-  "mflux==0.17.2",
+  "mflux==0.17.2; sys_platform == 'darwin'",
   "python-multipart>=0.0.21",
   "msgspec>=0.19.0",
   "zstandard>=0.23.0",
   "mlx-vlm>=0.3.11",
   "transformers>=5.0.0,<5.4.0",
+  "pydantic-settings>=2.13.1",
 ]
 
 [project.scripts]
@@ -47,11 +47,20 @@ dev = [
   "ruff>=0.11.13",
 ]
 
-# mlx[cuda] requires a newer version of mlx. the ideal on linux is: default to mlx[cpu] unless[cuda] specified.
 [project.optional-dependencies]
-# cuda = [
-#     "mlx[cuda]==0.26.3",
-# ]
+build = ["nanobind"]
+cpu = [
+  "mlx-cpu==0.31.1; sys_platform == 'linux'",
+  "torch>=2.10.0; sys_platform == 'linux'",
+]
+cuda12 = [
+  "mlx-cuda-12==0.31.1; sys_platform == 'linux'",
+  "torch>=2.10.0; sys_platform == 'linux'",
+]
+cuda13 = [
+  "mlx-cuda-13==0.31.1; sys_platform == 'linux'",
+  "torch>=2.10.0; sys_platform == 'linux'",
+]
 
 ###
 # workspace configuration
@@ -61,12 +70,30 @@ dev = [
 members = ["rust/exo_pyo3_bindings", "bench"]
 
 [tool.uv.sources]
-exo_pyo3_bindings = { workspace = true }
+exo-pyo3-bindings = { workspace = true }
 mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
 mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/fix-arrayscache-leak" }
-# Uncomment to use local mlx/mlx-lm development versions:
-# mlx = { path = "/Users/Shared/mlx", editable=true }
-# mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }
+torch = [
+  { index = "pytorch-cu130", marker = "sys_platform == 'linux' and extra == 'cuda13' and extra != 'cpu' and extra != 'cuda12'" },
+  { index = "pytorch-cu120", marker = "sys_platform == 'linux' and extra == 'cuda12' and extra != 'cpu' and extra != 'cuda13'" },
+  { index = "pytorch-cpu", marker = "sys_platform == 'linux' and extra == 'cpu' and extra != 'cuda12' and extra != 'cuda13'" },
+]
+vllm = { git = "https://github.com/hmellor/vllm.git", branch = "transformers-v5" }
+
+[[tool.uv.index]]
+name = "pytorch-cu130"
+url = "https://download.pytorch.org/whl/cu130"
+explicit = true
+
+[[tool.uv.index]]
+name = "pytorch-cu120"
+url = "https://download.pytorch.org/whl/cu120"
+explicit = true
+
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
 
 [build-system]
 requires = ["uv_build>=0.8.9,<0.9.0"]
@@ -77,7 +104,7 @@ build-backend = "uv_build"
 ###
 
 [tool.basedpyright]
-include = [".venv/lib/mlx", ".venv/lib/mlx_lm", "src", "bench"]
+include = ["src", "bench"]
 typeCheckingMode = "strict"
 failOnWarnings = true
 
@@ -104,9 +131,14 @@ exclude = [
 ]
 stubPath = ".mlx_typings"
 
+[[tool.basedpyright.executionEnvironments]]
+root = "src/exo/worker/engines/image"
+reportMissingModuleSource = false
+
 [[tool.basedpyright.executionEnvironments]]
 root = "src"
 
+
 ###
 # uv configuration
 ###
@@ -116,7 +148,46 @@ root = "src"
 required-version = ">=0.8.6"
 prerelease = "allow"
 environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
-extra-build-dependencies = { "miniaudio" = ["setuptools", "cffi", "pycparser"] }
+conflicts = [[{ extra = "cuda12" }, { extra = "cuda13" }, { extra = "cpu" }]]
+constraint-dependencies = ["transformers>=5.0.0,<5.4.0"]
+
+[tool.uv.extra-build-dependencies]
+miniaudio = ["setuptools", "cffi", "pycparser"]
+mlx = [
+  "setuptools",
+  "typing-extensions",
+  "nanobind",
+  "pybind11",
+  "wheel",
+  "cmake",
+  "ninja",
+]
+mlx-lm = ["setuptools"]
+xgrammar = [
+  "nanobind",
+  "setuptools",
+  "scikit-build-core",
+  "packaging",
+  "pathspec",
+]
+rouge-score = ["setuptools"]
+sacrebleu = ["setuptools"]
+sqlitedict = ["setuptools"]
+word2number = ["setuptools"]
+vllm = [
+  "setuptools",
+  "setuptools-scm",
+  "scikit-build-core",
+  "jinja2",
+  "wheel",
+  "markupsafe",
+  "typing-extensions",
+  "torch",
+]
+fastsafetensors = ["setuptools", "pybind11"]
+torch = ["typing-extensions"]
+torchvision = ["torch"]
+torchaudio = ["torch"]
 
 ###
 # ruff configuration
@@ -124,7 +195,6 @@ extra-build-dependencies = { "miniaudio" = ["setuptools", "cffi", "pycparser"] }
 
 [tool.ruff]
 extend-exclude = [
-  "shared/protobufs/**",
   "*mlx_typings/**",
   "rust/exo_pyo3_bindings/**",
   "bench/vendor/**",

python/parts.nix

@@ -1,18 +1,37 @@
 { inputs, ... }:
-{
-  perSystem =
-    { config, self', pkgs, lib, system, ... }:
+let
+  # Load workspace from uv.lock
+  workspace = inputs.uv2nix.lib.workspace.loadWorkspace {
+    workspaceRoot = ../.;
+  };
+
+  mkPythonSet = { pkgs, lib, self' }:
     let
-      # Load workspace from uv.lock
-      workspace = inputs.uv2nix.lib.workspace.loadWorkspace {
-        workspaceRoot = inputs.self;
-      };
-
-      # Create overlay from workspace
-      # Use wheels from PyPI for most packages; we override mlx with our pure Nix Metal build
-      overlay = workspace.mkPyprojectOverlay { sourcePreference = "wheel"; };
-
-      # Override overlay to inject Nix-built components
+      inherit (pkgs.stdenv.hostPlatform) isLinux isDarwin isx86_64;
+      inherit (pkgs.config) cudaSupport;
+      inherit (pkgs) cudaPackages;
+      cuda13Support = cudaSupport && cudaPackages.cudaMajorVersion == "13";
+      libmlx_source = if cuda13Support then "mlx-cuda-13" else if cudaSupport then "mlx-cuda-12" else "mlx-cpu";
+      uv_extra = if cuda13Support then "cuda13" else if cudaSupport then "cuda12" else "cpu";
+      python = pkgs.python313;
+      cudaLibs = with cudaPackages; [
+        cuda_cudart
+        cuda_cccl
+        cuda_cupti
+        cuda_nvrtc
+        cuda_nvtx
+        cudnn
+        libcufile
+        libcublas
+        libcufft
+        libcurand
+        libcusolver
+        libcusparse
+        libcusparse_lt
+        libnvjitlink
+        libnvshmem
+        nccl
+      ];
       exoOverlay = final: prev: {
         # Replace workspace exo_pyo3_bindings with Nix-built wheel.
         # Preserve passthru so mkVirtualEnv can resolve dependency groups.
@@ -32,126 +51,157 @@
           '';
         };
       };
+      buildSystemsOverlay = final: prev: { } //
+        lib.optionalAttrs isDarwin
+          {
+            mlx = prev.mlx.overrideAttrs (old:
+              let
+                # Static dependencies included directly during compilation
+                gguf-tools = pkgs.fetchFromGitHub {
+                  owner = "antirez";
+                  repo = "gguf-tools";
+                  rev = "8fa6eb65236618e28fd7710a0fba565f7faa1848";
+                  hash = "sha256-15FvyPOFqTOr5vdWQoPnZz+mYH919++EtghjozDlnSA=";
+                };
 
-      python = pkgs.python313;
+                metal_cpp = pkgs.fetchzip {
+                  url = "https://developer.apple.com/metal/cpp/files/metal-cpp_26.zip";
+                  hash = "sha256-7n2eI2lw/S+Us6l7YPAATKwcIbRRpaQ8VmES7S8ZjY8=";
+                };
 
-      # Overlay to provide build systems and custom packages
-      buildSystemsOverlay = final: prev: {
-        # mlx-lm is a git dependency that needs setuptools
-        mlx-lm = prev.mlx-lm.overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
-            final.setuptools
-          ];
+                nanobind = pkgs.fetchFromGitHub {
+                  owner = "wjakob";
+                  repo = "nanobind";
+                  rev = "v2.10.2";
+                  hash = "sha256-io44YhN+VpfHFWyvvLWSanRgbzA0whK8WlDNRi3hahU=";
+                  fetchSubmodules = true;
+                };
+              in
+              {
+                nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.cmake self'.packages.metal-toolchain ];
+                # TODO: non-sdk_26 support
+                buildInputs = (old.buildInputs or [ ])
+                  ++ [ gguf-tools pkgs.fmt pkgs.nlohmann_json pkgs.apple-sdk_26 ];
+                patches = [
+                  (pkgs.replaceVars ../nix/darwin-build-fixes.patch {
+                    sdkVersion = pkgs.apple-sdk_26.version;
+                    inherit (self'.packages.metal-toolchain) metalVersion;
+                  })
+                ];
+                postPatch = ''
+                  substituteInPlace mlx/backend/cpu/jit_compiler.cpp \
+                    --replace-fail "g++" "${lib.getExe' pkgs.stdenv.cc "c++"}"
+                '';
+
+                DEV_RELEASE = 1;
+                CMAKE_ARGS = toString ([
+                  (lib.cmakeBool "USE_SYSTEM_FMT" true)
+                  (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_GGUFLIB" "${gguf-tools}")
+                  (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_JSON" "${pkgs.nlohmann_json.src}")
+                  (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_NANOBIND" "${nanobind}")
+                  (lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true)
+                  (lib.cmakeBool "MLX_BUILD_CPU" true)
+                  (lib.cmakeBool "MLX_BUILD_METAL" true)
+                  (lib.cmakeOptionType "string" "CMAKE_INSTALL_LIBDIR" "lib")
+                  (lib.cmakeOptionType "filepath" "FETCHCONTENT_SOURCE_DIR_METAL_CPP" "${metal_cpp}")
+                  (lib.cmakeOptionType "string" "CMAKE_OSX_DEPLOYMENT_TARGET" "${pkgs.apple-sdk_26.version}")
+                  (lib.cmakeOptionType "filepath" "CMAKE_OSX_SYSROOT" "${pkgs.apple-sdk_26.passthru.sdkroot}")
+                ] ++ lib.optionals (isDarwin && isx86_64) [
+                  (lib.cmakeBool "MLX_ENABLE_X64_MAC" true)
+                ]);
+                SDKROOT = pkgs.apple-sdk_26.passthru.sdkroot;
+                MACOSX_DEPLOYMENT_TARGET = pkgs.apple-sdk_26.version;
+              });
+          } // lib.optionalAttrs isLinux {
+        mlx = prev.mlx.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ lib.optionals cudaSupport cudaLibs;
+          autoPatchelfIgnoreMissingDeps = lib.optionals cudaSupport [ "libcuda.so.1" ];
+          postInstall = (old.postInstall or "") + ''
+            cp -r "${final.${libmlx_source}}/${final.python.sitePackages}/mlx" "$out/${final.python.sitePackages}/mlx/"
+          '';
         });
-        # rouge-score and sacrebleu don't declare setuptools as a build dependency
-        rouge-score = prev.rouge-score.overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
-            final.setuptools
-          ];
+      } // lib.optionalAttrs cudaSupport {
+        "${libmlx_source}" = prev."${libmlx_source}".overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ cudaLibs;
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
         });
-        sacrebleu = prev.sacrebleu.overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
-            final.setuptools
-          ];
+        nvidia-cufile = prev.nvidia-cufile.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ [ pkgs.rdma-core ];
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
         });
-        sqlitedict = prev.sqlitedict.overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
-            final.setuptools
-          ];
+        nvidia-cusolver = prev.nvidia-cusolver.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ cudaLibs;
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
         });
-        word2number = prev.word2number.overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
-            final.setuptools
-          ];
+        nvidia-nvshmem-cu13 = prev.nvidia-nvshmem-cu13.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ [ pkgs.rdma-core pkgs.pmix pkgs.libfabric pkgs.ucx pkgs.openmpi ];
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
+        });
+        nvidia-cusparse = prev.nvidia-cusparse.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ [ cudaLibs ];
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
+        });
+        torch = prev.torch.overrideAttrs (old: {
+          buildInputs = old.buildInputs ++ cudaLibs;
+          autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
         });
-      } // lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin {
-        # Use our pure Nix-built MLX with Metal support (macOS only)
-        mlx = self'.packages.mlx;
       };
-
-      # Additional overlay for Linux-specific fixes (type checking env).
-      # Native wheels have shared lib dependencies we don't need at type-check time.
-      linuxOverlay = final: prev:
-        let
-          ignoreMissing = drv: drv.overrideAttrs { autoPatchelfIgnoreMissingDeps = [ "*" ]; };
-          nvidiaPackages = lib.filterAttrs (name: _: lib.hasPrefix "nvidia-" name) prev;
-        in
-        lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux (
-          (lib.mapAttrs (_: ignoreMissing) nvidiaPackages) // {
-            mlx = ignoreMissing prev.mlx;
-            mlx-cuda-13 = prev.mlx-cuda-13.overrideAttrs (old: {
-              buildInputs = (old.buildInputs or [ ]) ++ [
-                final.nvidia-cublas
-                final.nvidia-cuda-nvrtc
-                final.nvidia-cudnn-cu13
-                final.nvidia-nccl-cu13
-              ];
-              preFixup = ''
-                addAutoPatchelfSearchPath ${final.nvidia-cublas}
-                addAutoPatchelfSearchPath ${final.nvidia-cuda-nvrtc}
-                addAutoPatchelfSearchPath ${final.nvidia-cudnn-cu13}
-                addAutoPatchelfSearchPath ${final.nvidia-nccl-cu13}
-              '';
-              autoPatchelfIgnoreMissingDeps = [ "libcuda.so.1" ];
-            });
-            torch = ignoreMissing prev.torch;
-            triton = ignoreMissing prev.triton;
-          }
-        );
-
+      pyprojectOverlay = workspace.mkPyprojectOverlay {
+        sourcePreference = "wheel";
+        dependencies = { exo = [ uv_extra ]; exo-bench = [ ]; };
+      };
+      editableOverlay = workspace.mkEditablePyprojectOverlay {
+        # Use environment variable pointing to editable root directory
+        root = "$REPO_ROOT";
+        members = [ "exo" "exo-bench" ];
+      };
       pythonSet = (pkgs.callPackage inputs.pyproject-nix.build.packages {
         inherit python;
       }).overrideScope (
         lib.composeManyExtensions [
           inputs.pyproject-build-systems.overlays.default
-          overlay
+          pyprojectOverlay
           exoOverlay
           buildSystemsOverlay
-          linuxOverlay
         ]
       );
-      # mlx-cpu and mlx-cuda-13 both ship mlx/ site-packages files; keep first.
-      # mlx-cpu/mlx-cuda-13 and nvidia-cudnn-cu12/cu13 ship overlapping files.
-      venvCollisionPaths = lib.optionals pkgs.stdenv.hostPlatform.isLinux [
-        "lib/python3.13/site-packages/mlx*"
-        "lib/python3.13/site-packages/nvidia*"
-      ];
 
-      # Exclude bench deps from main env (bench has its own benchVenv)
-      exoDeps = removeAttrs workspace.deps.default [ "exo-bench" ];
-
-      exoVenv = (pythonSet.mkVirtualEnv "exo-env" exoDeps).overrideAttrs {
-        venvIgnoreCollisions = venvCollisionPaths;
-      };
-
-      # Virtual environment with dev dependencies for testing
-      testVenv = (pythonSet.mkVirtualEnv "exo-test-env" (
-        exoDeps // {
-          exo = [ "dev" ]; # Include pytest, pytest-asyncio, pytest-env
-        }
-      )).overrideAttrs {
-        venvIgnoreCollisions = venvCollisionPaths;
-      };
-
-      mkPythonScript = name: path: pkgs.writeShellApplication {
+      mkApp = cmd: name: members: pkgs.writeShellApplication {
         inherit name;
-        runtimeInputs = [ exoVenv ];
         runtimeEnv = {
           EXO_DASHBOARD_DIR = self'.packages.dashboard;
           EXO_RESOURCES_DIR = inputs.self + /resources;
         };
-        text = ''exec python ${path} "$@"'';
+        runtimeInputs = [
+          # mlx and mlx-cuda ship clashing cmake files - we dont need them at runtime anyway
+          ((pythonSet.mkVirtualEnv "${name}-env" members).overrideAttrs (_: { venvSkip = [ "lib/python${python.pythonVersion}/site-packages/mlx/share/cmake/*" ]; }))
+        ]
+        ++ lib.optionals isDarwin [ pkgs.macmon ];
+        text = "exec " + lib.optionalString cudaSupport "${lib.getExe pkgs.nix-gl-host} " + cmd;
+      };
+    in
+    {
+      inherit pythonSet;
+      editablePythonSet = pythonSet.overrideScope editableOverlay;
+      mkPythonScript = members: name: path: mkApp ''python ${path} "$@"'' name members;
+      mkExo = name: members: mkApp ''exo "$@"'' name members;
+    };
+in
+{
+  perSystem =
+    { self', pkgs, unfreePkgs, lib, ... }:
+    let
+      inherit (pkgs.stdenv.hostPlatform) isLinux;
+      inherit (mkPythonSet { inherit self' pkgs lib; }) pythonSet editablePythonSet mkPythonScript mkExo;
+
+      exoVenv = pythonSet.mkVirtualEnv "exo-env" { exo = lib.optionals isLinux [ "cpu" ]; };
+
+      # Virtual environment with dev dependencies for testing
+      testVenv = pythonSet.mkVirtualEnv "exo-test-env" {
+        exo = [ "dev" ] ++ lib.optionals isLinux [ "cpu" ]; # Include pytest, pytest-asyncio, pytest-env
       };
 
-      benchVenv = pythonSet.mkVirtualEnv "exo-bench-env" {
-        exo-bench = [ ];
-      };
-
-      mkBenchScript = name: path: pkgs.writeShellApplication {
-        inherit name;
-        runtimeInputs = [ benchVenv ];
-        text = ''exec python ${path} "$@"'';
-      };
+      mkBenchScript = mkPythonScript { exo-bench = [ ]; };
 
       mkSimplePythonScript = name: path: pkgs.writeShellApplication {
         inherit name;
@@ -159,46 +209,32 @@
         text = ''exec python ${path} "$@"'';
       };
 
-      exoPackage = pkgs.runCommand "exo"
-        {
-          nativeBuildInputs = [ pkgs.makeWrapper ];
-        }
-        ''
-          mkdir -p $out/bin
-
-          # Create wrapper script
-          makeWrapper ${exoVenv}/bin/exo $out/bin/exo \
-            --set EXO_DASHBOARD_DIR ${self'.packages.dashboard} \
-            --set EXO_RESOURCES_DIR ${inputs.self + /resources} \
-            ${lib.optionalString pkgs.stdenv.hostPlatform.isDarwin "--prefix PATH : ${pkgs.macmon}/bin"}
-        '';
     in
     {
-      # Python package only available on macOS (requires MLX/Metal)
-      packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin
-        {
-          exo = exoPackage;
-          # Test environment for running pytest outside of Nix sandbox (needs GPU access)
-          exo-test-env = testVenv;
-        } // {
-
-        inherit python;
-
+      packages = {
+        exo = mkExo "exo" { exo = lib.optionals isLinux [ "cpu" ]; };
+        # for devShell
+        exo-venv = exoVenv;
+        editableVenv = editablePythonSet.mkVirtualEnv "exo-dev-env" { exo = [ "dev" ]; };
+        # for running tests in ci
+        exo-test-env = testVenv;
         exo-bench = mkBenchScript "exo-bench" (inputs.self + /bench/exo_bench.py);
         exo-eval = mkBenchScript "exo-eval" (inputs.self + /bench/exo_eval.py);
         exo-eval-tool-calls = mkBenchScript "exo-eval-tool-calls" (inputs.self + /bench/eval_tool_calls.py);
+        # used by ./tests/run_exo_on.sh
         exo-get-all-models-on-cluster = mkSimplePythonScript "exo-get-all-models-on-cluster" (inputs.self + /tests/get_all_models_on_cluster.py);
+      } // lib.optionalAttrs isLinux {
+        exo-cuda-12 = (mkPythonSet { inherit self' lib; inherit (unfreePkgs.pkgsCuda.cudaPackages_12) pkgs; }).mkExo "exo-cuda-12" { exo = [ "cuda12" ]; };
+        exo-cuda-13 = (mkPythonSet { inherit self' lib; inherit (unfreePkgs.pkgsCuda.cudaPackages_13) pkgs; }).mkExo "exo-cuda-13" { exo = [ "cuda13" ]; };
       };
 
       checks = {
-        # Ruff linting (works on all platforms)
         lint = pkgs.runCommand "ruff-lint" { } ''
           export RUFF_CACHE_DIR="$TMPDIR/ruff-cache"
           ${pkgs.ruff}/bin/ruff check ${inputs.self}
           touch $out
         '';
 
-        # Hermetic basedpyright type checking
         typecheck = pkgs.runCommand "typecheck"
           {
             nativeBuildInputs = [
@@ -209,7 +245,7 @@
           ''
             cd ${inputs.self}
             export HOME=$TMPDIR
-            basedpyright --pythonpath ${testVenv}/bin/python
+            basedpyright --pythonpath ${testVenv}/bin/python --project ${inputs.self}/pyproject.toml
             touch $out
           '';
       };

resources/inference_model_cards/mlx-community--DeepSeek-V3.1-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 405874409472
+
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/main/generation_config.json
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/discussions/19
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95

resources/inference_model_cards/mlx-community--DeepSeek-V3.1-8bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 765577920512
+
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/blob/main/generation_config.json
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.1/discussions/19
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 378086226621
+
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/generation_config.json
+# Source: https://docs.vllm.ai/projects/recipes/en/latest/DeepSeek/DeepSeek-V3_2.html
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-8bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 755957120916
+
+# Source: https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/generation_config.json
+# Source: https://docs.vllm.ai/projects/recipes/en/latest/DeepSeek/DeepSeek-V3_2.html
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.5-Air-8bit.toml

@@ -13,3 +13,8 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 122406567936
+
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.5-Air-bf16.toml

@@ -13,3 +13,8 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 229780750336
+
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-4bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 198556925568
+
+# Source: https://huggingface.co/zai-org/GLM-4.7
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-6bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 286737579648
+
+# Source: https://huggingface.co/zai-org/GLM-4.7
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-8bit-gs32.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 396963397248
+
+# Source: https://huggingface.co/zai-org/GLM-4.7
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-4bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 19327352832
+
+# Source: https://huggingface.co/zai-org/GLM-4.7-Flash
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-5bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 22548578304
+
+# Source: https://huggingface.co/zai-org/GLM-4.7-Flash
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-6bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 26843545600
+
+# Source: https://huggingface.co/zai-org/GLM-4.7-Flash
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-8bit.toml

@@ -13,3 +13,10 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 34359738368
+
+# Source: https://huggingface.co/zai-org/GLM-4.7-Flash
+# Source: https://unsloth.ai/docs/models/glm-4.7-flash
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-5-8bit.toml

@@ -13,3 +13,9 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 790517400864
+
+# Source: https://huggingface.co/zai-org/GLM-5
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-5-MXFP4-Q8.toml

@@ -13,3 +13,9 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 405478939008
+
+# Source: https://huggingface.co/zai-org/GLM-5
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--GLM-5-bf16.toml

@@ -13,3 +13,9 @@ context_length = 202752
 
 [storage_size]
 in_bytes = 1487822475264
+
+# Source: https://huggingface.co/zai-org/GLM-5
+# Source: https://docs.z.ai/api-reference/llm/chat-completion
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95

resources/inference_model_cards/mlx-community--Kimi-K2-Instruct-4bit.toml

@@ -13,3 +13,8 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 620622774272
+
+# Source: https://huggingface.co/moonshotai/Kimi-K2-Instruct
+# Source: https://platform.kimi.ai/docs/guide/kimi-k2-quickstart
+[sampling_defaults]
+temperature = 0.6

resources/inference_model_cards/mlx-community--Kimi-K2-Thinking.toml

@@ -13,3 +13,8 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 706522120192
+
+# Source: https://huggingface.co/moonshotai/Kimi-K2-Thinking
+# Source: https://platform.kimi.ai/docs/guide/use-kimi-k2-thinking-model
+[sampling_defaults]
+temperature = 1.0

resources/inference_model_cards/mlx-community--Kimi-K2.5.toml

@@ -19,3 +19,17 @@ image_token_id = 163605
 model_type = "kimi_vl"
 weights_repo = "davehind/Kimi-K2.5-vision"
 processor_repo = "moonshotai/Kimi-K2.5"
+
+# Source: https://deepwiki.com/MoonshotAI/Kimi-K2.5/3.7-recommended-parameters
+# Source: https://unsloth.ai/docs/models/kimi-k2.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+min_p = 0.01
+
+# Source: https://deepwiki.com/MoonshotAI/Kimi-K2.5/3.7-recommended-parameters
+# Source: https://unsloth.ai/docs/models/kimi-k2.5
+[sampling_defaults.non_thinking]
+temperature = 0.6
+top_p = 0.95
+min_p = 0.01

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-4bit.toml

@@ -12,3 +12,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 39688355840
+
+# Source: https://huggingface.co/RedHatAI/Llama-3.1-Nemotron-70B-Instruct-HF-FP8-dynamic
+# Source: https://deepinfra.com/nvidia/Llama-3.1-Nemotron-70B-Instruct/api
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-8bit.toml

@@ -12,3 +12,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 74964549632
+
+# Source: https://huggingface.co/RedHatAI/Llama-3.1-Nemotron-70B-Instruct-HF-FP8-dynamic
+# Source: https://deepinfra.com/nvidia/Llama-3.1-Nemotron-70B-Instruct/api
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-bf16.toml

@@ -12,3 +12,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 141107412992
+
+# Source: https://huggingface.co/RedHatAI/Llama-3.1-Nemotron-70B-Instruct-HF-FP8-dynamic
+# Source: https://deepinfra.com/nvidia/Llama-3.1-Nemotron-70B-Instruct/api
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-4bit.toml

@@ -12,3 +12,12 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 2538706944
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults.non_thinking]
+temperature = 0.0

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-8bit.toml

@@ -12,3 +12,12 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 4794980352
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults.non_thinking]
+temperature = 0.0

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-bf16.toml

@@ -12,3 +12,12 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 9025492992
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+
+# Source: https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-4B-v1.1
+[sampling_defaults.non_thinking]
+temperature = 0.0

resources/inference_model_cards/mlx-community--Llama-3.2-1B-Instruct-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 729808896
+
+# Source: https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Llama-3.2-1B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.2-3B-Instruct-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 1863319552
+
+# Source: https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Llama-3.2-3B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.2-3B-Instruct-8bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 3501195264
+
+# Source: https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Llama-3.2-3B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.3-70B-Instruct-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 40652242944
+
+# Source: https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Llama-3.3-70B-Instruct-8bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 76799803392
+
+# Source: https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Llama-3.3-70B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-70B-Instruct-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 40652242944
+
+# Source: https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Meta-Llama-3.1-70B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-4bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 4637851648
+
+# Source: https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-8bit.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 8954839040
+
+# Source: https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-bf16.toml

@@ -13,3 +13,9 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 16882073600
+
+# Source: https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Meta-Llama-3.1-8B-Instruct/blob/main/generation_config.json
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.9

resources/inference_model_cards/mlx-community--MiniMax-M2.1-3bit.toml

@@ -13,3 +13,10 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 100086644736
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.1
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.1
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.1-8bit.toml

@@ -13,3 +13,10 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 242986745856
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.1
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.1
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.5-4bit.toml

@@ -13,3 +13,10 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 128666664960
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.5
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.5-6bit.toml

@@ -13,3 +13,10 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 185826705408
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.5
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.5-8bit.toml

@@ -13,3 +13,10 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 242986745856
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.5
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7-4bit-mxfp4.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 121537496794
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7-4bit.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 128682598717
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7-5bit.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 157262619651
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7-6bit.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 185842639299
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7-8bit.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 243002680786
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--MiniMax-M2.7.toml

@@ -13,3 +13,11 @@ context_length = 196608
 
 [storage_size]
 in_bytes = 457492783366
+
+# Source: https://huggingface.co/MiniMaxAI/MiniMax-M2.7
+# Source: https://github.com/MiniMax-AI/MiniMax-M2.7
+# Source: https://unsloth.ai/docs/models/minimax-m27
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-4Bit.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 17775342336
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-5Bit.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 21721476864
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-6Bit.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 25667611392
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 33559880448
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-BF16.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 63155889408
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-MXFP4.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 16788808704
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.toml

@@ -12,3 +12,9 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 19323906944
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B
+# Source: https://unsloth.ai/docs/models/nemotron-3
+[sampling_defaults]
+temperature = 1.0
+top_p = 1.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-Nano-9B-v2-4bits.toml

@@ -12,3 +12,14 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 5002791936
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2
+# Source: https://build.nvidia.com/nvidia/nvidia-nemotron-nano-9b-v2/modelcard
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2
+# Source: https://build.nvidia.com/nvidia/nvidia-nemotron-nano-9b-v2/modelcard
+[sampling_defaults.non_thinking]
+temperature = 0.0

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-Nano-9B-v2-6bit.toml

@@ -12,3 +12,14 @@ context_length = 131072
 
 [storage_size]
 in_bytes = 7224298496
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2
+# Source: https://build.nvidia.com/nvidia/nvidia-nemotron-nano-9b-v2/modelcard
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+
+# Source: https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2
+# Source: https://build.nvidia.com/nvidia/nvidia-nemotron-nano-9b-v2/modelcard
+[sampling_defaults.non_thinking]
+temperature = 0.0

resources/inference_model_cards/mlx-community--Qwen3-0.6B-4bit.toml

@@ -13,3 +13,17 @@ context_length = 32768
 
 [storage_size]
 in_bytes = 342884352
+
+# Source: https://huggingface.co/Qwen/Qwen3-0.6B#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3-0.6B#best-practices
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-0.6B-8bit.toml

@@ -13,3 +13,17 @@ context_length = 32768
 
 [storage_size]
 in_bytes = 698351616
+
+# Source: https://huggingface.co/Qwen/Qwen3-0.6B#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3-0.6B#best-practices
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-235B-A22B-Instruct-2507-4bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 141733920768
+
+# Source: https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507#best-practices
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-235B-A22B-Instruct-2507-8bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 268435456000
+
+# Source: https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507#best-practices
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-30B-A3B-4bit.toml

@@ -13,3 +13,17 @@ context_length = 32768
 
 [storage_size]
 in_bytes = 17612931072
+
+# Source: https://huggingface.co/Qwen/Qwen3-30B-A3B#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3-30B-A3B#best-practices
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-30B-A3B-8bit.toml

@@ -13,3 +13,17 @@ context_length = 32768
 
 [storage_size]
 in_bytes = 33279705088
+
+# Source: https://huggingface.co/Qwen/Qwen3-30B-A3B#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3-30B-A3B#best-practices
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-Coder-480B-A35B-Instruct-4bit.toml

@@ -13,3 +13,11 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 289910292480
+
+# Source: https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct#best-practices
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+repetition_penalty = 1.05

resources/inference_model_cards/mlx-community--Qwen3-Coder-480B-A35B-Instruct-8bit.toml

@@ -13,3 +13,11 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 579820584960
+
+# Source: https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct#best-practices
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+repetition_penalty = 1.05

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-4bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 45644286500
+
+# Source: https://huggingface.co/mlx-community/Qwen3-Coder-Next-4bit/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-5bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 57657697020
+
+# Source: https://huggingface.co/mlx-community/Qwen3-Coder-Next-4bit/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-6bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 68899327465
+
+# Source: https://huggingface.co/mlx-community/Qwen3-Coder-Next-4bit/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-8bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 89357758772
+
+# Source: https://huggingface.co/mlx-community/Qwen3-Coder-Next-4bit/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-bf16.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 157548627945
+
+# Source: https://huggingface.co/mlx-community/Qwen3-Coder-Next-4bit/blob/main/generation_config.json
+# Source: https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 40

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Instruct-4bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 46976204800
+
+# Source: https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct#best-practices
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Instruct-8bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 88814387200
+
+# Source: https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct#best-practices
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Thinking-4bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 47080074240
+
+# Source: https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Thinking-8bit.toml

@@ -13,3 +13,10 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 88814387200
+
+# Source: https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking#best-practices
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0

resources/inference_model_cards/mlx-community--Qwen3-VL-4B-Instruct-4bit.toml

@@ -12,3 +12,12 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 3340000000
+
+# Source: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct#generation-hyperparameters
+# Source: https://unsloth.ai/docs/models/qwen3-how-to-run-and-fine-tune/qwen3-vl-how-to-run-and-fine-tune
+[sampling_defaults]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-4bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 69593314272
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-6bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 100120675296
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-8bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 130648036320
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-bf16.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 245125640160
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-122B-A10B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-27B-4bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 16054266848
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-27B-8bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 29500943328
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-27B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-2B-MLX-8bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 2662787264
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-9B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-9B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-35B-A3B-4bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 20391405152
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-35B-A3B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-35B-A3B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-35B-A3B-8bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 37721130592
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-35B-A3B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 1.0
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-35B-A3B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5

resources/inference_model_cards/mlx-community--Qwen3.5-397B-A17B-4bit.toml

@@ -13,3 +13,23 @@ context_length = 262144
 
 [storage_size]
 in_bytes = 223860768352
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-397B-A17B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults]
+temperature = 0.6
+top_p = 0.95
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 0.0
+
+# Source: https://huggingface.co/Qwen/Qwen3.5-397B-A17B#best-practices
+# Source: https://unsloth.ai/docs/models/qwen3.5
+[sampling_defaults.non_thinking]
+temperature = 0.7
+top_p = 0.8
+top_k = 20
+min_p = 0.0
+repetition_penalty = 1.0
+presence_penalty = 1.5