mirror/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2026-04-25 00:09:15 -04:00
Code Issues Packages Projects Releases Wiki Activity

set KV_CACHE_BITS to None to disable quantized kv cache

Browse Source
This commit is contained in:
Alex Cheema
2025-12-30 16:49:32 +00:00
committed by Evan Quiney
parent 8a6da58404
commit 31d4cd8409
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/exo/worker/engines/mlx/constants.py
View File

@@ -9,7 +9,7 @@ MAX_KV_SIZE: int | None = 3200
KEEP_KV_SIZE: int | None = 1600
QUANTIZE_MODEL_MODE: str | None = "affine"
CACHE_GROUP_SIZE: int = 64
KV_CACHE_BITS: int | None = 8
KV_CACHE_BITS: int | None = None
# TODO: We should really make this opt-in, but Kimi requires trust_remote_code=True
TRUST_REMOTE_CODE: bool = True