From 08114410bc934ed1d25a74d8e46d7e04f49c10f0 Mon Sep 17 00:00:00 2001 From: Aaron Pham <29749331+aarnphm@users.noreply.github.com> Date: Sun, 10 Dec 2023 18:09:25 -0500 Subject: [PATCH] fix(openai): logprobs when echo is enabled (#761) Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com> --- openllm-python/src/openllm/entrypoints/openai.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openllm-python/src/openllm/entrypoints/openai.py b/openllm-python/src/openllm/entrypoints/openai.py index ae4894f2..04f203a9 100644 --- a/openllm-python/src/openllm/entrypoints/openai.py +++ b/openllm-python/src/openllm/entrypoints/openai.py @@ -298,18 +298,22 @@ async def completions(req, llm): i = output.index delta_text = output.text token_ids = output.token_ids - top_logprobs = output.logprobs[previous_num_tokens[i]:] logprobs = None + top_logprobs = None + if request.logprobs is not None: + top_logprobs = output.logprobs[previous_num_tokens[i]:] if request.echo and not previous_echo[i]: if not echo_without_generation: delta_text = res.prompt + delta_text token_ids = res.prompt_token_ids + token_ids - top_logprobs = res.prompt_logprobs + top_logprobs + if top_logprobs: + top_logprobs = res.prompt_logprobs + top_logprobs else: delta_text = res.prompt token_ids = res.prompt_token_ids - top_logprobs = res.prompt_logprobs + if top_logprobs: + top_logprobs = res.prompt_logprobs previous_echo[i] = True if request.logprobs is not None: logprobs = create_logprobs(output.token_ids, output.logprobs[previous_num_tokens[i]:], request.logprobs, len(previous_texts[i]), llm=llm)