diff --git a/core/config/model_config.go b/core/config/model_config.go index bf2c9e8bf1..9ca0d66a40 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1409,6 +1409,62 @@ func (c *ModelConfig) ApplyReasoningEffort(requestEffort string) { } } +// ReasoningDisabled reports whether the effective chat-template toggle asks the +// model not to think. +// +// The backend resolves enable_thinking with a fixed precedence: the client's +// request metadata overrides the operator's reasoning config (also set by +// reasoning_effort=none through ApplyReasoningEffort), which overrides the model +// YAML chat_template_kwargs. Callers that inspect the extracted reasoning have +// to apply the same precedence and value coercion, or they disagree with the +// template: a request can re-enable thinking over reasoning.disable=true, and a +// model can disable it through chat_template_kwargs alone. +func (c *ModelConfig) ReasoningDisabled() bool { + if c == nil { + return false + } + // Build the metadata the backend forwards, then let the same resolver that + // gRPCPredictOpts uses apply the layering and the string-to-bool coercion. + meta := map[string]string{} + if c.ReasoningConfig.DisableReasoning != nil { + if *c.ReasoningConfig.DisableReasoning { + meta["enable_thinking"] = "false" + } else { + meta["enable_thinking"] = "true" + } + } + if raw, ok := c.RequestMetadata["enable_thinking"]; ok { + meta["enable_thinking"] = raw + } + value, ok := c.ResolveChatTemplateKwargs(meta)["enable_thinking"] + if !ok { + return false + } + return !chatTemplateTruthy(value) +} + +// chatTemplateTruthy reports how a jinja template reads a chat_template_kwargs +// value: bools and numbers follow Go truthiness, and any non-empty string is +// true. That last rule is why coerceChatTemplateKwarg turns metadata +// "true"/"false" into real booleans before they reach the template. +func chatTemplateTruthy(value any) bool { + switch v := value.(type) { + case nil: + return false + case bool: + return v + case string: + return v != "" + case int: + return v != 0 + case int64: + return v != 0 + case float64: + return v != 0 + } + return true +} + // coerceChatTemplateKwarg coerces a request-metadata string value for use as a // jinja chat_template_kwarg. "true"/"false" become real booleans (so a jinja // `{% if preserve_thinking %}` reads false correctly, since any non-empty string diff --git a/core/config/thinking_disabled_test.go b/core/config/thinking_disabled_test.go new file mode 100644 index 0000000000..5b8665c01c --- /dev/null +++ b/core/config/thinking_disabled_test.go @@ -0,0 +1,76 @@ +package config_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" +) + +// ReasoningDisabled decides whether the streaming reasoning extractor may +// prefill a synthetic thinking-start token. It has to resolve enable_thinking +// exactly like the backend: request metadata over the operator's reasoning +// config, over the model's chat_template_kwargs. +var _ = Describe("ModelConfig.ReasoningDisabled", func() { + It("is false when nothing disables reasoning", func() { + Expect((&config.ModelConfig{}).ReasoningDisabled()).To(BeFalse()) + }) + + It("is true when the operator disabled reasoning", func() { + disabled := true + cfg := &config.ModelConfig{} + cfg.ReasoningConfig.DisableReasoning = &disabled + Expect(cfg.ReasoningDisabled()).To(BeTrue()) + }) + + It("is false when the operator explicitly enabled reasoning", func() { + enabled := false + cfg := &config.ModelConfig{} + cfg.ReasoningConfig.DisableReasoning = &enabled + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) + + It("honors a client enable_thinking=false", func() { + cfg := &config.ModelConfig{RequestMetadata: map[string]string{"enable_thinking": "false"}} + Expect(cfg.ReasoningDisabled()).To(BeTrue()) + }) + + It("stays enabled for enable_thinking=true", func() { + cfg := &config.ModelConfig{RequestMetadata: map[string]string{"enable_thinking": "true"}} + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) + + It("stays enabled for an unparsable metadata value", func() { + cfg := &config.ModelConfig{RequestMetadata: map[string]string{"enable_thinking": "sometimes"}} + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) + + It("lets request metadata re-enable thinking over the operator's disable", func() { + disabled := true + cfg := &config.ModelConfig{ + RequestMetadata: map[string]string{"enable_thinking": "true"}, + } + cfg.ReasoningConfig.DisableReasoning = &disabled + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) + + It("honors the model chat_template_kwargs enable_thinking=false", func() { + cfg := &config.ModelConfig{ + ChatTemplateKwargs: map[string]any{"enable_thinking": false}, + } + Expect(cfg.ReasoningDisabled()).To(BeTrue()) + }) + + It("lets request metadata override the model chat_template_kwargs", func() { + cfg := &config.ModelConfig{ + ChatTemplateKwargs: map[string]any{"enable_thinking": false}, + RequestMetadata: map[string]string{"enable_thinking": "true"}, + } + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) + + It("tolerates a nil receiver", func() { + var cfg *config.ModelConfig + Expect(cfg.ReasoningDisabled()).To(BeFalse()) + }) +}) diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index e226d37b51..19e95266ba 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -947,10 +947,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator } else { template = predInput } - thinkingStartToken := reason.DetectThinkingStartToken(template, &config.ReasoningConfig) - if config.TemplateConfig.UseTokenizerTemplate { - thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &config.ReasoningConfig) - } + thinkingStartToken := thinkingStartTokenFor(config, template) xlog.Debug("Thinking start token", "thinkingStartToken", thinkingStartToken, "template", template) diff --git a/core/http/endpoints/openai/chat_stream_thinking_disabled_test.go b/core/http/endpoints/openai/chat_stream_thinking_disabled_test.go new file mode 100644 index 0000000000..4fcb4d97c2 --- /dev/null +++ b/core/http/endpoints/openai/chat_stream_thinking_disabled_test.go @@ -0,0 +1,104 @@ +package openai + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/pkg/model" +) + +// Regression test for issue #12939. qwen3's chat template guards with +// {% if enable_thinking %}, but detection scans the raw template without +// evaluating the condition. With streaming and no tools, prefilling that token +// made the extractor classify the whole answer as reasoning whenever the client +// sent metadata.enable_thinking=false. +var _ = Describe("processStream with thinking disabled (issue #12939)", func() { + // The marker is only rendered when the template condition is true; the raw + // string still contains it, which is what detection sees. + const conditionalThinkTemplate = "{% if enable_thinking %}{% endif %}" + + var origInference modelInferenceFunc + + BeforeEach(func() { origInference = backend.ModelInferenceFunc }) + AfterEach(func() { backend.ModelInferenceFunc = origInference }) + + streamAnswer := func(answer string) { + backend.ModelInferenceFunc = func( + ctx context.Context, s string, messages schema.Messages, + images, videos, audios []string, + loader *model.ModelLoader, c *config.ModelConfig, cl *config.ModelConfigLoader, + o *config.ApplicationConfig, + tokenCallback func(string, backend.TokenUsage) bool, + tools, toolChoice string, + logprobs, topLogprobs *int, + logitBias map[string]float64, + metadata map[string]string, + ) (func() (backend.LLMResponse, error), error) { + return func() (backend.LLMResponse, error) { + usage := backend.TokenUsage{Prompt: 1, Completion: 1} + tokenCallback(answer, usage) + return backend.LLMResponse{Response: answer, Usage: usage}, nil + }, nil + } + } + + run := func(answer string, metadata map[string]string) []schema.OpenAIResponse { + streamAnswer(answer) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + req := &schema.OpenAIRequest{Cancel: cancel, Context: ctx} + req.Model = "qwen3-4b" + cfg := &config.ModelConfig{RequestMetadata: metadata} + responses := make(chan schema.OpenAIResponse, 16) + _, err := processStream(conditionalThinkTemplate, req, cfg, nil, config.NewApplicationConfig(), nil, responses, "req-1", 0) + Expect(err).ToNot(HaveOccurred()) + return drainChannel(responses) + } + + deltas := func(chunks []schema.OpenAIResponse) (reasoning, content string) { + for _, chunk := range chunks { + if len(chunk.Choices) == 0 || chunk.Choices[0].Delta == nil { + continue + } + if chunk.Choices[0].Delta.Reasoning != nil { + reasoning += *chunk.Choices[0].Delta.Reasoning + } + // Message.Content is an any: the streaming workers set it to a + // *string, while a plain string is also valid. + switch c := chunk.Choices[0].Delta.Content.(type) { + case *string: + if c != nil { + content += *c + } + case string: + content += c + } + } + return reasoning, content + } + + It("keeps the answer in content when enable_thinking=false", func() { + reasoning, content := deltas(run("Hello!", map[string]string{"enable_thinking": "false"})) + Expect(content).To(Equal("Hello!")) + Expect(reasoning).To(BeEmpty()) + }) + + It("reproduces the misclassification without the request lever (control)", func() { + reasoning, content := deltas(run("Hello!", nil)) + Expect(reasoning).To(Equal("Hello!"), + "without enable_thinking=false the synthetic prefill classifies the answer as reasoning; "+ + "this is the behaviour the fix guards against") + Expect(content).To(BeEmpty()) + }) + + It("still extracts real thinking tags when the model ignores the toggle", func() { + reasoning, content := deltas(run("planHello!", map[string]string{"enable_thinking": "false"})) + Expect(reasoning).To(Equal("plan")) + Expect(content).To(Equal("Hello!")) + }) +}) diff --git a/core/http/endpoints/openai/chat_stream_workers.go b/core/http/endpoints/openai/chat_stream_workers.go index f013b39a7d..8320bf4b7f 100644 --- a/core/http/endpoints/openai/chat_stream_workers.go +++ b/core/http/endpoints/openai/chat_stream_workers.go @@ -112,6 +112,26 @@ func chooseDeferredReasoning( return extractor.Reasoning() } +// thinkingStartTokenFor returns the thinking-start token to use for this +// request, or "" when the request disables thinking. +// +// The detectors scan the raw prompt/template without evaluating jinja +// conditionals, so qwen3's is found even when the surrounding +// {% if enable_thinking %} guard will not render it. Prefilling it in that case +// makes the extractor classify the whole answer as reasoning (issue #12939), +// so detection is skipped when the request asks for no thinking. Real +// … blocks that a model emits anyway are still extracted from +// the content itself. +func thinkingStartTokenFor(cfg *config.ModelConfig, template string) string { + if cfg.ReasoningDisabled() { + return "" + } + if cfg.TemplateConfig.UseTokenizerTemplate { + return reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) + } + return reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) +} + // processStream is the streaming worker for chat completions with no // tool/function calling involved. It pushes SSE-shaped chunks onto // `responses` and returns the authoritative cumulative TokenUsage from @@ -149,10 +169,7 @@ func processStream( } else { template = s } - thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) - if cfg.TemplateConfig.UseTokenizerTemplate { - thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) - } + thinkingStartToken := thinkingStartTokenFor(cfg, template) extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) // preferAutoparser is sticky: once the C++ autoparser has ever classified @@ -250,10 +267,7 @@ func processStreamWithTools( } else { template = prompt } - thinkingStartToken := reason.DetectThinkingStartToken(template, &cfg.ReasoningConfig) - if cfg.TemplateConfig.UseTokenizerTemplate { - thinkingStartToken = reason.DetectThinkingStartTokenInTemplate(template, &cfg.ReasoningConfig) - } + thinkingStartToken := thinkingStartTokenFor(cfg, template) extractor := reason.NewReasoningExtractor(thinkingStartToken, cfg.ReasoningConfig) result := ""