skip nan logprobs on tokens (#1622)

sometimes we generate NaN logprobs for tokens, this causes pydantic
validation errors on the receiving end. in this case we should just not
send the logprob items
This commit is contained in:
Evan Quiney
2026-02-25 17:44:15 +00:00
committed by GitHub
parent e23c3a3026
commit c4e874e97d

View File

@@ -1,3 +1,4 @@
import math
import time
from copy import deepcopy
from typing import Callable, Generator, cast, get_args
@@ -248,6 +249,9 @@ def extract_top_logprobs(
for i in range(top_logprobs):
token_id = int(top_indices[i].item())
token_logprob = float(top_values[i].item())
if math.isnan(token_logprob):
continue
# Decode token ID to string
token_str = tokenizer.decode([token_id])
# Get byte representation