mirror of
https://github.com/mudler/LocalAI.git
synced 2026-06-05 15:26:14 -04:00
4a2cc64d07
The OpenAI `reasoning_effort` field only reached the prompt template; it never toggled the backend's thinking. Map it onto ReasoningConfig.DisableReasoning (which becomes the enable_thinking gRPC metadata) in the request merge, so reasoning_effort="none" disables reasoning per request: the use case from #10072 (run a single Qwen3-style model and turn reasoning off for low-latency tasks while keeping it on for others). Effort levels (minimal/low/medium/high) enable thinking unless the model config explicitly disabled it (reasoning.disable: true wins and is never re-enabled by a request); "none" always disables. Closes #10072 Assisted-by: Claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
78 lines
2.5 KiB
Go
78 lines
2.5 KiB
Go
package backend
|
|
|
|
import (
|
|
"encoding/json"
|
|
|
|
"github.com/mudler/LocalAI/core/config"
|
|
"github.com/mudler/LocalAI/pkg/reasoning"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("grpcModelOpts EngineArgs", func() {
|
|
It("serialises engine_args as JSON preserving nested values", func() {
|
|
threads := 1
|
|
cfg := config.ModelConfig{
|
|
Threads: &threads,
|
|
LLMConfig: config.LLMConfig{
|
|
EngineArgs: map[string]any{
|
|
"data_parallel_size": 8,
|
|
"enable_expert_parallel": true,
|
|
"speculative_config": map[string]any{
|
|
"method": "ngram",
|
|
"num_speculative_tokens": 4,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
opts := grpcModelOpts(cfg, "/tmp/models")
|
|
Expect(opts.EngineArgs).NotTo(BeEmpty())
|
|
|
|
var round map[string]any
|
|
Expect(json.Unmarshal([]byte(opts.EngineArgs), &round)).To(Succeed())
|
|
Expect(round["data_parallel_size"]).To(BeEquivalentTo(8))
|
|
Expect(round["enable_expert_parallel"]).To(BeTrue())
|
|
Expect(round["speculative_config"]).To(HaveKeyWithValue("method", "ngram"))
|
|
})
|
|
|
|
It("leaves EngineArgs empty when unset", func() {
|
|
threads := 1
|
|
opts := grpcModelOpts(config.ModelConfig{Threads: &threads}, "/tmp/models")
|
|
Expect(opts.EngineArgs).To(BeEmpty())
|
|
})
|
|
})
|
|
|
|
// Guards the DisableReasoning -> enable_thinking metadata conversion that the
|
|
// per-request reasoning_effort feature (issue #10072) relies on: the request
|
|
// merge sets ReasoningConfig.DisableReasoning, and gRPCPredictOpts is where it
|
|
// becomes the gRPC PredictOptions.Metadata the backend reads.
|
|
var _ = Describe("gRPCPredictOpts enable_thinking metadata", func() {
|
|
// withReasoning builds a fully-defaulted config (gRPCPredictOpts dereferences
|
|
// many pointer fields) and overrides only the reasoning toggle.
|
|
withReasoning := func(disable *bool) config.ModelConfig {
|
|
cfg := config.ModelConfig{}
|
|
cfg.SetDefaults()
|
|
cfg.ReasoningConfig = reasoning.Config{DisableReasoning: disable}
|
|
return cfg
|
|
}
|
|
disabled := true
|
|
enabled := false
|
|
|
|
It("emits enable_thinking=false when reasoning is disabled", func() {
|
|
opts := gRPCPredictOpts(withReasoning(&disabled), "/tmp/models")
|
|
Expect(opts.Metadata).To(HaveKeyWithValue("enable_thinking", "false"))
|
|
})
|
|
|
|
It("emits enable_thinking=true when reasoning is enabled", func() {
|
|
opts := gRPCPredictOpts(withReasoning(&enabled), "/tmp/models")
|
|
Expect(opts.Metadata).To(HaveKeyWithValue("enable_thinking", "true"))
|
|
})
|
|
|
|
It("omits enable_thinking when reasoning is unset", func() {
|
|
opts := gRPCPredictOpts(withReasoning(nil), "/tmp/models")
|
|
Expect(opts.Metadata).ToNot(HaveKey("enable_thinking"))
|
|
})
|
|
})
|