From 44383528b5a66711e9aa63430f4c95a14bee77d6 Mon Sep 17 00:00:00 2001 From: Aaron Pham <29749331+aarnphm@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:19:01 -0500 Subject: [PATCH] fix(logprobs): correct check logprobs (#779) * fix(logprobs): correct check logprobs Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com> * chore: update changlog Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com> --------- Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com> --- changelog.d/779.fix.md | 1 + openllm-python/src/openllm/_runners.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/779.fix.md diff --git a/changelog.d/779.fix.md b/changelog.d/779.fix.md new file mode 100644 index 00000000..690e453b --- /dev/null +++ b/changelog.d/779.fix.md @@ -0,0 +1 @@ +Fixes logprobs branch with PyTorch backend. diff --git a/openllm-python/src/openllm/_runners.py b/openllm-python/src/openllm/_runners.py index 2b5de8f4..23af330e 100644 --- a/openllm-python/src/openllm/_runners.py +++ b/openllm-python/src/openllm/_runners.py @@ -162,7 +162,7 @@ class PyTorchRunnable(bentoml.Runnable): input_len = len(prompt_token_ids) if self.is_encoder_decoder: - if config['logprobs'] > 0: # FIXME: logprobs is not supported + if config['logprobs']: # FIXME: logprobs is not supported raise NotImplementedError('Logprobs is yet to be supported with encoder-decoder models.') encoder_output = self.model.encoder(input_ids=torch.as_tensor([prompt_token_ids], device=self.device))[0] start_ids = torch.as_tensor(