mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-13 14:56:11 -04:00
pos-ei-don
4894056380
Qwen3-style chat templates append the opening <think> tag to the *prompt*
when thinking is enabled. The model therefore never generates it and emits
only the reasoning text plus the closing </think>.
sglang's ReasoningParser keys off the opening tag:
in_reasoning = self._in_reasoning or self.think_start_token in text
if not in_reasoning:
return StreamingParseResult(normal_text=text)
so with such a template the entire completion — reasoning and answer, the
raw </think> in between — is returned as content and reasoning_content
stays empty, no matter how reasoning_parser is configured.
sglang's own OpenAI server handles this via
force_reasoning = (self.template_manager.force_reasoning
or self._get_reasoning_from_request(request))
This backend has no template manager, so derive the same signal from the
rendered prompt: if it ends with the detector's think_start_token, the tag
was prefilled and the parser is constructed with force_reasoning=True.
Structured decoding is the exception, and it matters: a grammar applies
from the first token, so the model cannot emit the closing tag even though
the template opened the block. The whole completion is schema output and
belongs in content — forcing there files it as reasoning and returns an
empty answer. Measured against a JSON-schema code audit: 10107 characters
of "reasoning", zero content. sglang's own server keeps the two apart for
the same reason; its grammar backend owns the reasoning prefix when a
reasoning parser is configured.
force_reasoning is only passed when it is meant to be True, so detector
defaults (DeepSeek-R1 already defaults to True) are untouched, and a
prompt without a prefilled tag behaves exactly as before — which matters,
because forcing unconditionally makes an answer generated with thinking
off disappear into reasoning_content.
The construction is factored into _new_reasoning_parser() so the streaming
and non-streaming paths, which previously built the parser separately,
cannot drift apart.
Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com>
222 lines
9.1 KiB
Python
222 lines
9.1 KiB
Python
"""Unit tests for the sglang backend.
|
|
|
|
Helper-level tests run without launching the gRPC server or loading model
|
|
weights — they only exercise the pure-Python helpers on
|
|
``BackendServicer``. They do still require ``sglang`` to be importable
|
|
because ``_apply_engine_args`` validates keys against
|
|
``ServerArgs``'s dataclass fields.
|
|
"""
|
|
import unittest
|
|
|
|
|
|
class TestSglangHelpers(unittest.TestCase):
|
|
"""Tests for the pure helpers on BackendServicer (no gRPC, no engine)."""
|
|
|
|
def _servicer(self):
|
|
import sys
|
|
import os
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from backend import BackendServicer # noqa: E402
|
|
return BackendServicer()
|
|
|
|
def test_parse_options(self):
|
|
servicer = self._servicer()
|
|
opts = servicer._parse_options([
|
|
"tool_parser:hermes",
|
|
"reasoning_parser:deepseek_r1",
|
|
"invalid_no_colon",
|
|
"key_with_colons:a:b:c",
|
|
])
|
|
self.assertEqual(opts["tool_parser"], "hermes")
|
|
self.assertEqual(opts["reasoning_parser"], "deepseek_r1")
|
|
self.assertEqual(opts["key_with_colons"], "a:b:c")
|
|
self.assertNotIn("invalid_no_colon", opts)
|
|
|
|
def test_apply_engine_args_known_keys(self):
|
|
"""User-supplied JSON merges into the kwargs dict; pre-set typed
|
|
fields stay put when not overridden."""
|
|
import json as _json
|
|
servicer = self._servicer()
|
|
base = {
|
|
"model_path": "facebook/opt-125m",
|
|
"mem_fraction_static": 0.7,
|
|
}
|
|
extras = _json.dumps({
|
|
"trust_remote_code": True,
|
|
"speculative_algorithm": "EAGLE",
|
|
"speculative_num_steps": 1,
|
|
})
|
|
out = servicer._apply_engine_args(base, extras)
|
|
self.assertIs(out, base) # in-place merge — same dict back
|
|
self.assertTrue(out["trust_remote_code"])
|
|
self.assertEqual(out["speculative_algorithm"], "EAGLE")
|
|
self.assertEqual(out["speculative_num_steps"], 1)
|
|
self.assertEqual(out["model_path"], "facebook/opt-125m")
|
|
self.assertEqual(out["mem_fraction_static"], 0.7)
|
|
|
|
def test_apply_engine_args_engine_args_overrides_typed_fields(self):
|
|
"""engine_args wins over previously-set typed kwargs (vLLM precedence)."""
|
|
import json as _json
|
|
servicer = self._servicer()
|
|
base = {"model_path": "facebook/opt-125m", "mem_fraction_static": 0.7}
|
|
out = servicer._apply_engine_args(
|
|
base, _json.dumps({"mem_fraction_static": 0.5}),
|
|
)
|
|
self.assertEqual(out["mem_fraction_static"], 0.5)
|
|
|
|
def test_apply_engine_args_unknown_key_raises(self):
|
|
"""Typo'd key raises ValueError with a close-match suggestion."""
|
|
import json as _json
|
|
servicer = self._servicer()
|
|
base = {"model_path": "facebook/opt-125m"}
|
|
with self.assertRaises(ValueError) as ctx:
|
|
servicer._apply_engine_args(
|
|
base, _json.dumps({"trust_remotecode": True}),
|
|
)
|
|
msg = str(ctx.exception)
|
|
self.assertIn("trust_remotecode", msg)
|
|
self.assertIn("trust_remote_code", msg)
|
|
|
|
def test_apply_engine_args_empty_passthrough(self):
|
|
"""Empty / None engine_args returns the kwargs dict untouched."""
|
|
servicer = self._servicer()
|
|
base = {"model_path": "facebook/opt-125m"}
|
|
self.assertIs(servicer._apply_engine_args(base, ""), base)
|
|
self.assertIs(servicer._apply_engine_args(base, None), base)
|
|
|
|
def test_apply_engine_args_invalid_json_raises(self):
|
|
servicer = self._servicer()
|
|
with self.assertRaises(ValueError) as ctx:
|
|
servicer._apply_engine_args({}, "not-json")
|
|
self.assertIn("not valid JSON", str(ctx.exception))
|
|
|
|
def test_apply_engine_args_non_object_raises(self):
|
|
servicer = self._servicer()
|
|
with self.assertRaises(ValueError) as ctx:
|
|
servicer._apply_engine_args({}, "[1,2,3]")
|
|
self.assertIn("must be a JSON object", str(ctx.exception))
|
|
|
|
def test_build_prompt_forwards_enable_thinking(self):
|
|
from types import SimpleNamespace
|
|
|
|
class Tok:
|
|
def __init__(self):
|
|
self.kwargs = None
|
|
|
|
def apply_chat_template(self, messages, **kwargs):
|
|
self.kwargs = kwargs
|
|
return "PROMPT"
|
|
|
|
def kwargs_for(metadata):
|
|
servicer = self._servicer()
|
|
tok = Tok()
|
|
servicer.tokenizer = tok
|
|
msg = SimpleNamespace(
|
|
role="user", content="hi", name="",
|
|
tool_call_id="", reasoning_content="", tool_calls="",
|
|
)
|
|
req = SimpleNamespace(
|
|
Prompt="", UseTokenizerTemplate=True,
|
|
Messages=[msg], Tools="", Metadata=metadata,
|
|
)
|
|
self.assertEqual(servicer._build_prompt(req), "PROMPT")
|
|
return tok.kwargs
|
|
|
|
self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
|
|
# "false" used to be dropped, so Qwen3 kept thinking on
|
|
self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
|
|
self.assertNotIn("enable_thinking", kwargs_for({}))
|
|
self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
|
|
|
|
def test_reasoning_parser_forced_when_template_prefills_think_tag(self):
|
|
"""Qwen3's template puts ``<think>`` in the prompt, so the completion
|
|
never contains it. Without force_reasoning the detector treats the whole
|
|
completion as normal text and reasoning_content stays empty."""
|
|
servicer = self._servicer()
|
|
servicer.reasoning_parser_name = "qwen3"
|
|
|
|
# What the model actually emits when the prompt ends in "<think>".
|
|
completion = "adding two and two</think>4"
|
|
|
|
forced = servicer._new_reasoning_parser(False, prompt="user: hi\n<think>\n")
|
|
reasoning, content = forced.parse_non_stream(completion)
|
|
self.assertEqual(reasoning, "adding two and two")
|
|
self.assertEqual(content, "4")
|
|
|
|
# No prefilled tag in the prompt: detector default, unchanged behaviour.
|
|
unforced = servicer._new_reasoning_parser(False, prompt="user: hi\n")
|
|
reasoning, content = unforced.parse_non_stream(completion)
|
|
self.assertFalse(reasoning)
|
|
self.assertEqual(content, completion)
|
|
|
|
def test_reasoning_parser_not_forced_when_thinking_is_off(self):
|
|
"""Thinking off means no ``<think>`` in the prompt either, so the answer
|
|
must not be swallowed into reasoning_content."""
|
|
servicer = self._servicer()
|
|
servicer.reasoning_parser_name = "qwen3"
|
|
|
|
parser = servicer._new_reasoning_parser(False, prompt="user: primes?\n")
|
|
reasoning, content = parser.parse_non_stream("2,3,5,7,11")
|
|
self.assertFalse(reasoning)
|
|
self.assertEqual(content, "2,3,5,7,11")
|
|
|
|
def test_grammar_constrained_output_is_not_forced_into_reasoning(self):
|
|
"""Structured decoding applies from the first token, so the model cannot
|
|
emit the closing tag even though the template opened the block. The whole
|
|
completion is schema output and must stay in content."""
|
|
servicer = self._servicer()
|
|
servicer.reasoning_parser_name = "qwen3"
|
|
|
|
schema_out = '{"findings": [{"line": 42, "issue": "off-by-one"}]}'
|
|
parser = servicer._new_reasoning_parser(
|
|
False, prompt="audit this\n<think>\n", grammar_constrained=True,
|
|
)
|
|
reasoning, content = parser.parse_non_stream(schema_out)
|
|
self.assertFalse(reasoning)
|
|
self.assertEqual(content, schema_out)
|
|
|
|
def test_reasoning_parser_absent_without_configured_parser(self):
|
|
servicer = self._servicer()
|
|
servicer.reasoning_parser_name = None
|
|
self.assertIsNone(servicer._new_reasoning_parser(False, prompt="<think>"))
|
|
|
|
def test_explicit_zero_temperature_and_seed_are_preserved(self):
|
|
"""Temperature=0 is greedy decoding and 0 is a valid seed — neither is
|
|
an unset value. A dropped seed turns a reproducible request random."""
|
|
from types import SimpleNamespace
|
|
|
|
servicer = self._servicer()
|
|
import sys as _sys
|
|
_SEED_KEY_FOR_TEST = _sys.modules["backend"]._SEED_KEY
|
|
request = SimpleNamespace(
|
|
Temperature=0,
|
|
N=0,
|
|
PresencePenalty=0,
|
|
FrequencyPenalty=0,
|
|
RepetitionPenalty=0,
|
|
TopP=0,
|
|
TopK=0,
|
|
MinP=0,
|
|
Seed=0,
|
|
StopPrompts=[],
|
|
StopTokenIds=[],
|
|
IgnoreEOS=False,
|
|
Tokens=0,
|
|
MinTokens=0,
|
|
SkipSpecialTokens=False,
|
|
Grammar="",
|
|
)
|
|
|
|
params = servicer._build_sampling_params(request)
|
|
self.assertEqual(params["temperature"], 0)
|
|
self.assertEqual(params[_SEED_KEY_FOR_TEST], 0)
|
|
# Other protobuf-default scalar fields must remain filtered. top_k=0 in
|
|
# particular is not a value sglang accepts (-1 disables it), so it must
|
|
# keep falling through to the engine default.
|
|
self.assertNotIn("top_p", params)
|
|
self.assertNotIn("top_k", params)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|