fix(cli): make sure to pass the dtype to subprocess service (#628)

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
This commit is contained in:
Aaron Pham
2023-11-13 05:32:17 -05:00
committed by GitHub
parent 099c0dc31b
commit 852cd863a9

View File

@@ -633,6 +633,7 @@ def process_environ(
'OPENLLM_SERIALIZATION': serialisation,
'OPENLLM_BACKEND': llm.__llm_backend__,
'OPENLLM_CONFIG': config.model_dump_json(flatten=True).decode(),
'TORCH_DTYPE': str(llm._torch_dtype).split('.')[-1]
}
)
if llm.quantise: