diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index fec738fb1..c6669def6 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -186,6 +186,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if not sep or not level: print(f"effort_budgets entry {entry!r} is not level=tokens, ignoring it", file=sys.stderr) continue + if not raw.strip(): + print(f"effort_budgets entry {entry!r} has no token count, ignoring it", file=sys.stderr) + continue tokens = cls._parse_thinking_budget(raw) if tokens is not None: budgets[level] = tokens @@ -349,8 +352,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # read it: every effort level got the same fixed `thinking_budget`, so # low/medium/high produced identical reasoning. With this map a request # gets the budget of its effort level; levels absent from the map, and - # requests without an effort, keep `thinking_budget`. Same semantics as - # vLLM's proposed `--reasoning-effort-budgets` (vllm-project/vllm#55432). + # requests without an effort, keep `thinking_budget`. self.effort_budgets: dict = self._parse_effort_budgets(opts.get("effort_budgets")) # Model-level default for whether the chat template opens a reasoning @@ -525,7 +527,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): def _thinking_default(self, request) -> Optional[bool]: """Whether this request should render with reasoning on, off, or unset. - Per-request ``Metadata["enable_thinking"]`` wins; the model-level + Per-request ``Metadata["enable_thinking"]`` wins; next, an effort level + listed in ``effort_budgets`` turns reasoning on; the model-level ``reasoning_default`` option fills in when the request is silent. Returns None when neither says anything, leaving template behaviour untouched. @@ -534,10 +537,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if wanted in ("true", "false"): return wanted == "true" # An effort level listed in `effort_budgets` is an explicit request to - # think. The Go side only sets enable_thinking for the levels it knows - # (minimal/low/medium/high); for others such as `xhigh` or `max` it sets - # nothing, so `reasoning_default: off` would win and asking for MORE - # reasoning would yield NONE. + # think. Normally the Go side already sends enable_thinking for known + # levels; this covers levels it does not set it for (custom names in + # the map, or callers that only send reasoning_effort), where + # `reasoning_default: off` would otherwise win. An explicit + # enable_thinking=false above still takes precedence. if self.effort_budgets and self._effort(request) in self.effort_budgets: return True if self.reasoning_default == "off": diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 0ff300683..8a251573d 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -361,24 +361,37 @@ class TestSglangHelpers(unittest.TestCase): self.assertEqual(servicer._parse_effort_budgets("low=abc;medium;=5;high=0;xhigh=100"), {"xhigh": 100}) self.assertEqual(servicer._parse_effort_budgets(None), {}) self.assertEqual(servicer._parse_effort_budgets(""), {}) + self.assertEqual(servicer._parse_effort_budgets(";;low=512;"), {"low": 512}) + + def test_effort_budgets_empty_value_warns(self): + """`high=` is skipped like any other invalid entry, and says so.""" + import contextlib + import io + + servicer = self._servicer() + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(servicer._parse_effort_budgets("high=;max=16384"), {"max": 16384}) + self.assertIn("'high='", err.getvalue()) def test_effort_level_selects_its_budget(self): """The request's reasoning_effort (Metadata, forwarded by the Go side) picks its own budget, so low/medium/high stop being identical.""" servicer = self._servicer() servicer.thinking_budget = 256 - servicer.effort_budgets = {"low": 512, "medium": 2048, "high": 8192} - for level, budget in (("low", 512), ("medium", 2048), ("HIGH", 8192)): + servicer.effort_budgets = {"low": 512, "medium": 2048, "high": 8192, "xhigh": 16384, "max": 32768} + for level, budget in (("low", 512), ("medium", 2048), ("HIGH", 8192), ("xhigh", 16384), ("max", 32768)): params = servicer._build_sampling_params(self._sampling_request({"reasoning_effort": level})) self.assertEqual(params["custom_params"], {"thinking_budget": budget}, level) def test_listed_effort_turns_thinking_on_despite_default_off(self): - """`xhigh`/`max` get no enable_thinking from the Go side; if the level is - in effort_budgets it must still think instead of falling to default off.""" + """A request that carries only reasoning_effort (no enable_thinking) for a + level listed in effort_budgets must think instead of falling to default off.""" servicer = self._servicer() servicer.reasoning_default = "off" - servicer.effort_budgets = {"xhigh": 16384} + servicer.effort_budgets = {"xhigh": 16384, "max": 16384} self.assertTrue(servicer._thinking_default(_request(metadata={"reasoning_effort": "xhigh"}))) + self.assertTrue(servicer._thinking_default(_request(metadata={"reasoning_effort": "max"}))) self.assertFalse(servicer._thinking_default(_request(metadata={"reasoning_effort": "ultra"}))) self.assertFalse(servicer._thinking_default( _request(metadata={"reasoning_effort": "xhigh", "enable_thinking": "false"}))) diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index d5daadd43..d987fcf9c 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -1105,7 +1105,7 @@ options: Entries are separated by `;` and use the same parsing rules as `thinking_budget` (invalid entries are skipped with a warning). A request whose effort is not listed falls back to `thinking_budget`; without either, no budget is sent. -A level listed here also turns reasoning on for that request, even when `reasoning_default:off` is set. +A level listed here also turns reasoning on for that request, even when `reasoning_default:off` is set; an explicit `enable_thinking: "false"` in the request metadata (also sent when the model sets `reasoning.disable: true`) still wins. The budget is an upper bound, not a target: the model may stop thinking earlier. `reasoning_default:on` or `reasoning_default:off` sets the default for LocalAI's tokenizer chat template.