diff --git a/openllm-python/src/_openllm_tiny/_llm.py b/openllm-python/src/_openllm_tiny/_llm.py
index 9a813728..ed4f004c 100644
--- a/openllm-python/src/_openllm_tiny/_llm.py
+++ b/openllm-python/src/_openllm_tiny/_llm.py
@@ -172,9 +172,8 @@ class LLM:
     top_p = 1.0 if config['temperature'] <= 1e-5 else config['top_p']
     config = config.model_copy(update=dict(stop=list(stop), stop_token_ids=stop_token_ids, top_p=top_p))
 
-    sampling_params = SamplingParams(**{
-      k: getattr(config, k, None) for k in set(inspect.signature(SamplingParams).parameters.keys())
-    })
+    params = {k: getattr(config, k, None) for k in set(inspect.signature(SamplingParams).parameters.keys())}
+    sampling_params = SamplingParams(**{k: v for k, v in params.items() if v is not None})
 
     try:
       async for it in self._model.generate(