mirror of
https://github.com/exo-explore/exo.git
synced 2025-12-30 17:50:34 -05:00
Compare commits
1 Commits
alexcheema
...
alexcheema
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72fca71522 |
@@ -343,6 +343,10 @@ def make_kv_cache(
|
||||
) -> list[KVCache | RotatingKVCache | QuantizedKVCache]:
|
||||
assert hasattr(model, "layers")
|
||||
|
||||
if hasattr(model, "make_cache"):
|
||||
logger.info(f"Using make_cache")
|
||||
return model.make_cache() # type: ignore
|
||||
|
||||
if max_kv_size is None:
|
||||
if KV_CACHE_BITS is None:
|
||||
logger.info("Using default KV cache")
|
||||
|
||||
Reference in New Issue
Block a user