From 852cd863a96be330a85db26d602a75d33cfdee93 Mon Sep 17 00:00:00 2001
From: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
Date: Mon, 13 Nov 2023 05:32:17 -0500
Subject: [PATCH] fix(cli): make sure to pass the dtype to subprocess service
 (#628)

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
---
 openllm-python/src/openllm_cli/entrypoint.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openllm-python/src/openllm_cli/entrypoint.py b/openllm-python/src/openllm_cli/entrypoint.py
index 59369822..ec979c2f 100644
--- a/openllm-python/src/openllm_cli/entrypoint.py
+++ b/openllm-python/src/openllm_cli/entrypoint.py
@@ -633,6 +633,7 @@ def process_environ(
       'OPENLLM_SERIALIZATION': serialisation,
       'OPENLLM_BACKEND': llm.__llm_backend__,
       'OPENLLM_CONFIG': config.model_dump_json(flatten=True).decode(),
+      'TORCH_DTYPE': str(llm._torch_dtype).split('.')[-1]
     }
   )
   if llm.quantise: