mirror of
https://github.com/exo-explore/exo.git
synced 2026-01-21 12:30:22 -05:00
Compare commits
1 Commits
foo
...
alexcheema
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72fca71522 |
@@ -343,6 +343,10 @@ def make_kv_cache(
|
|||||||
) -> list[KVCache | RotatingKVCache | QuantizedKVCache]:
|
) -> list[KVCache | RotatingKVCache | QuantizedKVCache]:
|
||||||
assert hasattr(model, "layers")
|
assert hasattr(model, "layers")
|
||||||
|
|
||||||
|
if hasattr(model, "make_cache"):
|
||||||
|
logger.info(f"Using make_cache")
|
||||||
|
return model.make_cache() # type: ignore
|
||||||
|
|
||||||
if max_kv_size is None:
|
if max_kv_size is None:
|
||||||
if KV_CACHE_BITS is None:
|
if KV_CACHE_BITS is None:
|
||||||
logger.info("Using default KV cache")
|
logger.info("Using default KV cache")
|
||||||
|
|||||||
Reference in New Issue
Block a user