mirror/OpenLLM
1
0
Fork 0
mirror of https://github.com/bentoml/OpenLLM.git synced 2026-01-06 06:29:21 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(backend): correct use variable for backend when initialisation (#702)

Browse Source 
Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com>
This commit is contained in:
Aaron Pham
2023-11-19 22:42:25 -05:00
committed by GitHub
parent 44f05da845
commit 4491aa54d0
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openllm-python/src/openllm/_llm.py
View File

@@ -217,7 +217,7 @@ class LLM(t.Generic[M, T], ReprMixin):
def _resolve_quantise(self, quantise, backend):
if backend in ('pt', 'vllm'):return quantise
if backend=='ctranslate':return self._resolve_ctranslate_quantise(quantise)
raise NotImplementedError(f"Quantisation is not supported for backend '{self.__llm_backend__}'")
raise NotImplementedError(f"Quantisation is not supported for backend '{backend}'")
def _resolve_ctranslate_quantise(self,quantise):
if quantise in {'int4', 'awq', 'gptq', 'squeezellm'}:raise ValueError(f"Quantisation '{quantise}' is not supported for backend 'ctranslate'")
if quantise == 'int8':quantise='int8_float16' if self._has_gpus else 'int8_float32'