mirror/OpenLLM
1
0
Fork 0
mirror of https://github.com/bentoml/OpenLLM.git synced 2026-02-07 22:33:28 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: make sure to only update fields when correct type is parse

Browse Source 
Signed-off-by: paperspace <29749331+aarnphm@users.noreply.github.com>
This commit is contained in:
paperspace
2024-05-14 06:42:46 +00:00
parent 03c51ab133
commit e9246e7772
1 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
openllm-python/src/_openllm_tiny/_service.py
View File

@@ -72,7 +72,11 @@ class LLMService:
request_id: t.Optional[str] = None,
llm_config: t.Dict[str, t.Any] = pydantic.Field(default=LLMConfig, description='LLM Config'),
) -> core.GenerationOutput:
llm_config.update(stop=stop, stop_token_ids=stop_token_ids)
if stop:
llm_config.update(stop=stop)
if stop_token_ids:
llm_config.update(stop_token_ids=stop_token_ids)
return await self.llm.generate(
prompt=prompt, prompt_token_ids=prompt_token_ids, request_id=request_id, **llm_config
)
@@ -87,7 +91,10 @@ class LLMService:
request_id: t.Optional[str] = None,
llm_config: t.Dict[str, t.Any] = pydantic.Field(default=LLMConfig, description='LLM Config'),
) -> t.AsyncGenerator[str, None]:
llm_config.update(stop=stop, stop_token_ids=stop_token_ids)
if stop:
llm_config.update(stop=stop)
if stop_token_ids:
llm_config.update(stop_token_ids=stop_token_ids)
_config = LLMConfig.model_construct_env(**core.utils.dict_filter_none(llm_config))
previous_texts = [''] * _config['n']