diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 8bddd002a5..6d4d851038 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -391,6 +391,40 @@ func extractToolChoiceFunctionName(m map[string]any) string { return "" } +// mergeChatTemplateKwargs folds a request's chat_template_kwargs (the +// llama-server / vLLM / SGLang field) into its metadata map, so clients written +// for those servers can set e.g. enable_thinking per request. Scalars become the +// strings the metadata path expects (booleans as "true"/"false", which +// ResolveChatTemplateKwargs turns back into booleans); nested objects and arrays +// are skipped, as is the reserved "chat_template_kwargs" key. An explicit +// metadata entry wins over the same key here. With no kwargs, metadata is +// returned unchanged. +func mergeChatTemplateKwargs(kwargs map[string]any, metadata map[string]string) map[string]string { + if len(kwargs) == 0 { + return metadata + } + out := make(map[string]string, len(kwargs)+len(metadata)) + for k, v := range kwargs { + if k == "chat_template_kwargs" { + continue + } + switch t := v.(type) { + case bool: + out[k] = strconv.FormatBool(t) + case string: + out[k] = t + case float64: + out[k] = strconv.FormatFloat(t, 'f', -1, 64) + case json.Number: + out[k] = t.String() + } + } + for k, v := range metadata { + out[k] = v + } + return out +} + func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.OpenAIRequest) error { // Discard client tools before anything below derives functions or a forced // tool_choice from them; the model's own MCP tools are injected later. @@ -456,8 +490,9 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema. // Forward the client's request metadata so chat-template kwargs set per-request // (enable_thinking, reasoning_effort, preserve_thinking, ...) reach the backend // and override the model's reasoning-config defaults. See gRPCPredictOpts. - if len(input.Metadata) > 0 { - config.RequestMetadata = input.Metadata + // The llama-server style chat_template_kwargs field takes the same path. + if md := mergeChatTemplateKwargs(input.ChatTemplateKwargs, input.Metadata); len(md) > 0 { + config.RequestMetadata = md } // Collapse the modern max_completion_tokens alias into the diff --git a/core/http/middleware/request_test.go b/core/http/middleware/request_test.go index d2d1d539df..2d24a14e1a 100644 --- a/core/http/middleware/request_test.go +++ b/core/http/middleware/request_test.go @@ -917,6 +917,41 @@ var _ = Describe("SetModelAndConfig metadata passthrough (chat completions)", fu Expect(*captured).ToNot(BeNil()) Expect((*captured).RequestMetadata).To(HaveKeyWithValue("preserve_thinking", "true")) }) + + It("folds llama-server style chat_template_kwargs into the request metadata", func() { + app, captured := buildApp() + body := `{"model":"test-model","messages":[{"role":"user","content":"hi"}],` + + `"chat_template_kwargs":{"enable_thinking":false,"reasoning_effort":"low","budget":512,` + + `"nested":{"a":1},"list":[1],"chat_template_kwargs":"x"}}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(*captured).ToNot(BeNil()) + Expect((*captured).RequestMetadata).To(Equal(map[string]string{ + "enable_thinking": "false", + "reasoning_effort": "low", + "budget": "512", + })) + }) + + It("lets an explicit metadata key win over chat_template_kwargs", func() { + app, captured := buildApp() + body := `{"model":"test-model","messages":[{"role":"user","content":"hi"}],` + + `"chat_template_kwargs":{"enable_thinking":false,"preserve_thinking":true},` + + `"metadata":{"enable_thinking":"true"}}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(*captured).ToNot(BeNil()) + Expect((*captured).RequestMetadata).To(HaveKeyWithValue("enable_thinking", "true")) + Expect((*captured).RequestMetadata).To(HaveKeyWithValue("preserve_thinking", "true")) + }) }) // A model config may default to decayed_mean pooling with a half-life diff --git a/core/schema/openai.go b/core/schema/openai.go index 62b2452621..2df9f9e8d9 100644 --- a/core/schema/openai.go +++ b/core/schema/openai.go @@ -263,6 +263,12 @@ type OpenAIRequest struct { ReasoningEffort string `json:"reasoning_effort,omitempty" yaml:"reasoning_effort"` Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata"` + + // ChatTemplateKwargs is the per-request chat_template_kwargs field that + // llama.cpp's llama-server, vLLM and SGLang accept (e.g. + // {"enable_thinking": false}). Scalar values are folded into Metadata, so + // they take the same path to the backend; an explicit Metadata key wins. + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty" yaml:"chat_template_kwargs"` } type ModelsDataResponse struct { diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 62153f3438..98e92d7d9f 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -751,6 +751,19 @@ and (for `enable_thinking` / `reasoning_effort`) takes effect across every backe reads them, not just llama.cpp. Typed (non-boolean) values are only supported through the model YAML `chat_template_kwargs`, where YAML preserves the type. +Clients written for llama.cpp's `llama-server`, vLLM or SGLang send the same switches as a +top-level `chat_template_kwargs` object. LocalAI accepts that too and folds its scalar values +into `metadata` (booleans become `"true"` / `"false"`, numbers become strings; nested objects +and arrays are ignored). If a key appears in both, `metadata` wins: + +```json +{ + "model": "qwen3", + "messages": [{"role": "user", "content": "hi"}], + "chat_template_kwargs": { "enable_thinking": false } +} +``` + ### Multimodal Backend Options | Option | Type | Default | Description | @@ -1134,8 +1147,9 @@ curl http://localhost:8080/v1/chat/completions \ **Priority order:** 1. Request-level `metadata.enable_thinking` (highest priority) -2. Model config `reasoning.disable` (fallback) -3. Auto-detected from model template (default) +2. Request-level `chat_template_kwargs.enable_thinking` (the llama-server / vLLM form) +3. Model config `reasoning.disable` (fallback) +4. Auto-detected from model template (default) ## Pipeline Configuration