From 073adc50c952c612196925b67713fcc5d1ecb7dd Mon Sep 17 00:00:00 2001 From: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:07:52 +0200 Subject: [PATCH] feat(sglang): map reasoning_effort to per-level thinking budgets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go side already forwards the request's effort as Metadata["reasoning_effort"], but the sglang backend only read enable_thinking: every level got the same model-level thinking_budget, so low/medium/high produced identical reasoning. - New model option `effort_budgets:low=…;medium=…;high=…` picks the budget of the request's level; unlisted levels and requests without an effort keep `thinking_budget`. Parsing reuses _parse_thinking_budget. - A level listed in effort_budgets turns reasoning on for that request, so levels the Go side does not know cannot silently disable it. - ApplyReasoningEffort now treats `xhigh` and `max` as thinking levels. Before, they left DisableReasoning unset, so with a model default of "off" asking for more reasoning produced none. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> (cherry picked from commit 879c686ab15ac3d4936a7ae5ba4adc46483576fe) --- backend/python/sglang/backend.py | 65 +++++++++++++++++++- backend/python/sglang/test.py | 61 ++++++++++++++++++ core/config/model_config.go | 2 +- core/config/reasoning_effort_test.go | 10 +++ docs/content/advanced/model-configuration.md | 4 +- docs/content/features/text-generation.md | 13 ++++ 6 files changed, 149 insertions(+), 6 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index b3a41a2ce..fec738fb1 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -122,6 +122,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): # tests that construct it directly) doesn't AttributeError in # _build_sampling_params. thinking_budget: Optional[int] = None + effort_budgets: Optional[dict] = None # {effort level: tokens}, from `effort_budgets` reasoning_default: Optional[str] = None def _parse_options(self, options_list) -> Dict[str, str]: @@ -165,6 +166,44 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): return None return int(number) + @classmethod + def _parse_effort_budgets(cls, value) -> dict: + """Parse `effort_budgets` ("low=512;medium=2048;high=8192") into {level: tokens}. + + `;` separates entries because the YAML `options:` list already uses `:` + between option name and value. Invalid entries are skipped with a warning + (same policy as `thinking_budget`: never fail LoadModel over an option). + """ + budgets: dict = {} + if value is None or str(value).strip() == "": + return budgets + for entry in str(value).split(";"): + entry = entry.strip() + if not entry: + continue + level, sep, raw = entry.partition("=") + level = level.strip().lower() + if not sep or not level: + print(f"effort_budgets entry {entry!r} is not level=tokens, ignoring it", file=sys.stderr) + continue + tokens = cls._parse_thinking_budget(raw) + if tokens is not None: + budgets[level] = tokens + return budgets + + @staticmethod + def _effort(request) -> str: + """The request's reasoning_effort as forwarded by the Go side, lower-cased.""" + metadata = getattr(request, "Metadata", None) or {} + return (metadata.get("reasoning_effort") or "").strip().lower() + + def _thinking_budget_for(self, request) -> Optional[int]: + """Budget for this request: its effort level's entry, else the model default.""" + effort = self._effort(request) + if effort and self.effort_budgets and effort in self.effort_budgets: + return self.effort_budgets[effort] + return self.thinking_budget + @staticmethod def _strict_thinking_warning(thinking_budget: Optional[int], engine_kwargs: dict) -> Optional[str]: """sglang only enforces the budget with enable_strict_thinking on; without @@ -304,6 +343,16 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): opts.get("thinking_budget") ) + # Per-effort budgets, e.g. `effort_budgets:low=512;medium=2048;high=8192`. + # The Go side already forwards the request's `reasoning_effort` as + # Metadata["reasoning_effort"], but until now nothing in this backend + # read it: every effort level got the same fixed `thinking_budget`, so + # low/medium/high produced identical reasoning. With this map a request + # gets the budget of its effort level; levels absent from the map, and + # requests without an effort, keep `thinking_budget`. Same semantics as + # vLLM's proposed `--reasoning-effort-budgets` (vllm-project/vllm#55432). + self.effort_budgets: dict = self._parse_effort_budgets(opts.get("effort_budgets")) + # Model-level default for whether the chat template opens a reasoning # block, as "off" or "on". Rides the same `options:` mechanism as # thinking_budget above. @@ -338,7 +387,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): print(f"engine_args error: {err}", file=sys.stderr) return backend_pb2.Result(success=False, message=str(err)) - warning = self._strict_thinking_warning(self.thinking_budget, engine_kwargs) + warning = self._strict_thinking_warning( + self.thinking_budget or max((self.effort_budgets or {}).values(), default=None), engine_kwargs + ) if warning: print(warning, file=sys.stderr) @@ -465,8 +516,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): except json.JSONDecodeError: sampling_params["ebnf"] = grammar - if self.thinking_budget is not None: - sampling_params["custom_params"] = {"thinking_budget": self.thinking_budget} + budget = self._thinking_budget_for(request) + if budget is not None: + sampling_params["custom_params"] = {"thinking_budget": budget} return sampling_params @@ -481,6 +533,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): wanted = request.Metadata.get("enable_thinking", "").lower() if wanted in ("true", "false"): return wanted == "true" + # An effort level listed in `effort_budgets` is an explicit request to + # think. The Go side only sets enable_thinking for the levels it knows + # (minimal/low/medium/high); for others such as `xhigh` or `max` it sets + # nothing, so `reasoning_default: off` would win and asking for MORE + # reasoning would yield NONE. + if self.effort_budgets and self._effort(request) in self.effort_budgets: + return True if self.reasoning_default == "off": return False if self.reasoning_default == "on": diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index f99774770..0ff300683 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -340,6 +340,67 @@ class TestSglangHelpers(unittest.TestCase): params = servicer._build_sampling_params(request) self.assertEqual(params["max_new_tokens"], 4000) + def _sampling_request(self, metadata=None): + from types import SimpleNamespace + + return SimpleNamespace( + Temperature=0.7, N=0, PresencePenalty=0, FrequencyPenalty=0, + RepetitionPenalty=0, TopP=0, TopK=0, MinP=0, Seed=0, + StopPrompts=[], StopTokenIds=[], IgnoreEOS=False, Tokens=0, + MinTokens=0, SkipSpecialTokens=False, Grammar="", + Metadata=metadata or {}, + ) + + def test_effort_budgets_parse(self): + """`low=512;medium=2048;high=8192` -> per-level dict; bad entries are skipped.""" + servicer = self._servicer() + self.assertEqual( + servicer._parse_effort_budgets("low=512; Medium=2048.0;high=8192"), + {"low": 512, "medium": 2048, "high": 8192}, + ) + self.assertEqual(servicer._parse_effort_budgets("low=abc;medium;=5;high=0;xhigh=100"), {"xhigh": 100}) + self.assertEqual(servicer._parse_effort_budgets(None), {}) + self.assertEqual(servicer._parse_effort_budgets(""), {}) + + def test_effort_level_selects_its_budget(self): + """The request's reasoning_effort (Metadata, forwarded by the Go side) + picks its own budget, so low/medium/high stop being identical.""" + servicer = self._servicer() + servicer.thinking_budget = 256 + servicer.effort_budgets = {"low": 512, "medium": 2048, "high": 8192} + for level, budget in (("low", 512), ("medium", 2048), ("HIGH", 8192)): + params = servicer._build_sampling_params(self._sampling_request({"reasoning_effort": level})) + self.assertEqual(params["custom_params"], {"thinking_budget": budget}, level) + + def test_listed_effort_turns_thinking_on_despite_default_off(self): + """`xhigh`/`max` get no enable_thinking from the Go side; if the level is + in effort_budgets it must still think instead of falling to default off.""" + servicer = self._servicer() + servicer.reasoning_default = "off" + servicer.effort_budgets = {"xhigh": 16384} + self.assertTrue(servicer._thinking_default(_request(metadata={"reasoning_effort": "xhigh"}))) + self.assertFalse(servicer._thinking_default(_request(metadata={"reasoning_effort": "ultra"}))) + self.assertFalse(servicer._thinking_default( + _request(metadata={"reasoning_effort": "xhigh", "enable_thinking": "false"}))) + + def test_effort_without_entry_falls_back_to_thinking_budget(self): + servicer = self._servicer() + servicer.thinking_budget = 256 + servicer.effort_budgets = {"high": 8192} + for metadata in ({}, {"reasoning_effort": "low"}, {"reasoning_effort": ""}): + params = servicer._build_sampling_params(self._sampling_request(metadata)) + self.assertEqual(params["custom_params"], {"thinking_budget": 256}, metadata) + + def test_effort_budgets_alone_without_default(self): + """Only levels in the map get a budget; others stay unconfigured (no key).""" + servicer = self._servicer() + servicer.effort_budgets = {"low": 512} + self.assertEqual( + servicer._build_sampling_params(self._sampling_request({"reasoning_effort": "low"}))["custom_params"], + {"thinking_budget": 512}, + ) + self.assertNotIn("custom_params", servicer._build_sampling_params(self._sampling_request({"reasoning_effort": "high"}))) + def test_thinking_budget_accepts_integral_spellings(self): """YAML options arrive as strings; "512" and "512.0" both mean 512.""" servicer = self._servicer() diff --git a/core/config/model_config.go b/core/config/model_config.go index 97845a6a9..82e12bca4 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1274,7 +1274,7 @@ func (c *ModelConfig) ApplyReasoningEffort(requestEffort string) { case "none": disable := true c.ReasoningConfig.DisableReasoning = &disable - case "minimal", "low", "medium", "high": + case "minimal", "low", "medium", "high", "xhigh", "max": if c.ReasoningConfig.DisableReasoning == nil || !*c.ReasoningConfig.DisableReasoning { enable := false c.ReasoningConfig.DisableReasoning = &enable diff --git a/core/config/reasoning_effort_test.go b/core/config/reasoning_effort_test.go index 8f23ba690..c3a341eb2 100644 --- a/core/config/reasoning_effort_test.go +++ b/core/config/reasoning_effort_test.go @@ -35,6 +35,16 @@ var _ = Describe("ModelConfig.ApplyReasoningEffort", func() { Expect(*c.ReasoningConfig.DisableReasoning).To(BeFalse()) }) + It("enables thinking for the levels above high", func() { + for _, level := range []string{"xhigh", "max"} { + c := &config.ModelConfig{} + c.ApplyReasoningEffort(level) + Expect(c.ReasoningEffort).To(Equal(level)) + Expect(c.ReasoningConfig.DisableReasoning).ToNot(BeNil()) + Expect(*c.ReasoningConfig.DisableReasoning).To(BeFalse()) + } + }) + It("does not let a level override an operator's config-level disable", func() { disabled := true c := &config.ModelConfig{} diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 5fd89323c..f014be7d0 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -686,7 +686,7 @@ These load-time options control how the backend parses `` reasoning block This is the load-time reasoning configuration. The orthogonal per-request `enable_thinking` chat-template kwarg toggles thinking on/off per call without restarting the model. It can be driven either by the YAML `reasoning.disable` field (model default) or per request via the OpenAI `reasoning_effort` field on `/v1/chat/completions`: - `reasoning_effort: "none"` disables thinking for that request (`enable_thinking=false`) - useful to run a single reasoning model like Qwen3 for low-latency tasks while still enabling reasoning on other requests. -- `reasoning_effort: "minimal" | "low" | "medium" | "high"` enables thinking, unless the model config explicitly set `reasoning.disable: true` (an operator's explicit disable wins and is never re-enabled by a request). +- `reasoning_effort: "minimal" | "low" | "medium" | "high" | "xhigh" | "max"` enables thinking, unless the model config explicitly set `reasoning.disable: true` (an operator's explicit disable wins and is never re-enabled by a request). {{% /notice %}} #### `reasoning_effort` as a chat-template kwarg @@ -697,7 +697,7 @@ Set a per-model default in the config so every request inherits it (a per-reques ```yaml name: my-model -reasoning_effort: none # none | minimal | low | medium | high +reasoning_effort: none # none | minimal | low | medium | high | xhigh | max ``` For [realtime pipelines]({{%relref "features/openai-realtime" %}}), set it on the pipeline so it applies to the pipeline's LLM without editing that model's own config: diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index fecd61eb0..d5daadd43 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -1095,6 +1095,19 @@ LocalAI warns if you configure a budget without that engine option. Keep the budget well below the `max_tokens` of your requests: if `max_tokens` is reached first, the budget never triggers and the whole reply can be spent on reasoning, leaving the answer empty. +`effort_budgets` maps the request's `reasoning_effort` to its own budget, so the levels differ in how long the model may think: + +```yaml +options: + - thinking_budget:256 # fallback: no effort, or a level not listed below + - effort_budgets:low=256;medium=1024;high=4096;xhigh=16384;max=16384 +``` + +Entries are separated by `;` and use the same parsing rules as `thinking_budget` (invalid entries are skipped with a warning). +A request whose effort is not listed falls back to `thinking_budget`; without either, no budget is sent. +A level listed here also turns reasoning on for that request, even when `reasoning_default:off` is set. +The budget is an upper bound, not a target: the model may stop thinking earlier. + `reasoning_default:on` or `reasoning_default:off` sets the default for LocalAI's tokenizer chat template. Request metadata `enable_thinking` set to `"true"` or `"false"` overrides this default. An explicit prompt bypasses tokenizer template rendering.