From b30a4123987f532dabd855952182522cecefe0db Mon Sep 17 00:00:00 2001
From: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
Date: Mon, 13 Nov 2023 23:05:27 -0500
Subject: [PATCH] fix(cli): set default dtype to auto infer (#642)

Signed-off-by: Aaron Pham <29749331+aarnphm@users.noreply.github.com>
---
 openllm-python/src/openllm_cli/_factory.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openllm-python/src/openllm_cli/_factory.py b/openllm-python/src/openllm_cli/_factory.py
index 40e06bf7..fff432c6 100644
--- a/openllm-python/src/openllm_cli/_factory.py
+++ b/openllm-python/src/openllm_cli/_factory.py
@@ -304,9 +304,9 @@ def machine_option(f: _AnyCallable | None = None, **attrs: t.Any) -> t.Callable[
 def dtype_option(f: _AnyCallable | None = None, **attrs: t.Any) -> t.Callable[[FC], FC]:
   return cli_option(
     '--dtype',
-    type=click.Choice(['float16', 'float32', 'bfloat16']),
+    type=click.Choice(['float16', 'float32', 'bfloat16', 'auto']),
     envvar='TORCH_DTYPE',
-    default='float16',
+    default='auto',
     help='Optional dtype for casting tensors for running inference.',
     **attrs,
   )(f)