diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index a6efe693c..a7933b911 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -679,6 +679,24 @@ class TestStreamingReasoningSplit(unittest.TestCase): THINK = ["Let me ", "count the ", "primes.", ""] ANSWER = ["There are ", "37 primes."] + def setUp(self): + from types import SimpleNamespace + from unittest.mock import patch + import backend + + def parser_request(model, messages, tools=None): + return SimpleNamespace( + model=model, messages=messages, + tools=[SimpleNamespace(function=SimpleNamespace(**tool["function"])) + for tool in tools] if tools else None, + ) + + # These fake-parser tests also run on backend profiles whose vLLM + # predates ParserManager and therefore has no _ParserRequest import. + request_patch = patch.object(backend, "_ParserRequest", parser_request, create=True) + request_patch.start() + self.addCleanup(request_patch.stop) + class _DeltaMsg: def __init__(self, content=None, reasoning=None, tool_calls=None): self.content = content diff --git a/docs/content/features/openai-functions.md b/docs/content/features/openai-functions.md index 5e7a384ca..d80a91767 100644 --- a/docs/content/features/openai-functions.md +++ b/docs/content/features/openai-functions.md @@ -84,6 +84,13 @@ When you import a vLLM model through the LocalAI gallery, the importer looks up Available tool parsers include `hermes`, `llama3_json`, `llama4_pythonic`, `mistral`, `qwen3_xml`, `deepseek_v3`, `granite4`, `kimi_k2`, `glm45`, and more. Available reasoning parsers include `deepseek_r1`, `qwen3`, `mistral`, `gemma4`, `granite`. See the upstream vLLM documentation for the full list. +With `backend: vllm`, vLLM 0.30 or later, and a configured reasoning parser, +streamed responses separate `delta.reasoning_content`, `delta.content`, and tool +calls as tokens arrive. The final chunk does not repeat the generated text. +If parsing fails during a stream, the backend buffers the remaining output and +sends any matching, unsent suffixes in the final chunk. Older vLLM versions keep +the previous streaming behavior and can include reasoning in `delta.content`. + ### MLX / MLX-VLM MLX backends **auto-detect** the right tool parser by inspecting the model's chat template - you don't need to set anything. Just load an MLX-quantized model that was trained with tool support: