From 23839b74a20fc3833fc1fb8350a5e27efb467ae0 Mon Sep 17 00:00:00 2001 From: Cristi An Date: Fri, 30 Oct 2026 08:55:41 +0000 Subject: [PATCH] feat(api): accept request-level chat_template_kwargs Clients written for llama-server, vLLM or SGLang switch template variables per request with a top-level chat_template_kwargs object, most often {"enable_thinking": false}. OpenAIRequest had no such field, so it was dropped: with a Qwen3-style template thinking stayed on, the reply landed in message.reasoning and content came back empty. Fold the field's scalar values into the request metadata, so it takes the path metadata already takes (gRPCPredictOpts, ResolveChatTemplateKwargs). Booleans become "true"/"false", strings pass through, numbers become their decimal string; nested values are skipped. An explicit metadata key wins. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Cristi An (cherry picked from commit 2cd7425e871a0994970e2c490d9e17f5a283917c) (cherry picked from commit 5e86ad6bd5286cc8db398d278adaa32e562223ca) --- core/http/middleware/request.go | 39 +++++++++++++++++++- core/http/middleware/request_test.go | 35 ++++++++++++++++++ core/schema/openai.go | 6 +++ docs/content/advanced/model-configuration.md | 18 ++++++++- 4 files changed, 94 insertions(+), 4 deletions(-) diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 8bddd002a5..6d4d851038 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -391,6 +391,40 @@ func extractToolChoiceFunctionName(m map[string]any) string { return "" } +// mergeChatTemplateKwargs folds a request's chat_template_kwargs (the +// llama-server / vLLM / SGLang field) into its metadata map, so clients written +// for those servers can set e.g. enable_thinking per request. Scalars become the +// strings the metadata path expects (booleans as "true"/"false", which +// ResolveChatTemplateKwargs turns back into booleans); nested objects and arrays +// are skipped, as is the reserved "chat_template_kwargs" key. An explicit +// metadata entry wins over the same key here. With no kwargs, metadata is +// returned unchanged. +func mergeChatTemplateKwargs(kwargs map[string]any, metadata map[string]string) map[string]string { + if len(kwargs) == 0 { + return metadata + } + out := make(map[string]string, len(kwargs)+len(metadata)) + for k, v := range kwargs { + if k == "chat_template_kwargs" { + continue + } + switch t := v.(type) { + case bool: + out[k] = strconv.FormatBool(t) + case string: + out[k] = t + case float64: + out[k] = strconv.FormatFloat(t, 'f', -1, 64) + case json.Number: + out[k] = t.String() + } + } + for k, v := range metadata { + out[k] = v + } + return out +} + func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.OpenAIRequest) error { // Discard client tools before anything below derives functions or a forced // tool_choice from them; the model's own MCP tools are injected later. @@ -456,8 +490,9 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema. // Forward the client's request metadata so chat-template kwargs set per-request // (enable_thinking, reasoning_effort, preserve_thinking, ...) reach the backend // and override the model's reasoning-config defaults. See gRPCPredictOpts. - if len(input.Metadata) > 0 { - config.RequestMetadata = input.Metadata + // The llama-server style chat_template_kwargs field takes the same path. + if md := mergeChatTemplateKwargs(input.ChatTemplateKwargs, input.Metadata); len(md) > 0 { + config.RequestMetadata = md } // Collapse the modern max_completion_tokens alias into the diff --git a/core/http/middleware/request_test.go b/core/http/middleware/request_test.go index d2d1d539df..2d24a14e1a 100644 --- a/core/http/middleware/request_test.go +++ b/core/http/middleware/request_test.go @@ -917,6 +917,41 @@ var _ = Describe("SetModelAndConfig metadata passthrough (chat completions)", fu Expect(*captured).ToNot(BeNil()) Expect((*captured).RequestMetadata).To(HaveKeyWithValue("preserve_thinking", "true")) }) + + It("folds llama-server style chat_template_kwargs into the request metadata", func() { + app, captured := buildApp() + body := `{"model":"test-model","messages":[{"role":"user","content":"hi"}],` + + `"chat_template_kwargs":{"enable_thinking":false,"reasoning_effort":"low","budget":512,` + + `"nested":{"a":1},"list":[1],"chat_template_kwargs":"x"}}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(*captured).ToNot(BeNil()) + Expect((*captured).RequestMetadata).To(Equal(map[string]string{ + "enable_thinking": "false", + "reasoning_effort": "low", + "budget": "512", + })) + }) + + It("lets an explicit metadata key win over chat_template_kwargs", func() { + app, captured := buildApp() + body := `{"model":"test-model","messages":[{"role":"user","content":"hi"}],` + + `"chat_template_kwargs":{"enable_thinking":false,"preserve_thinking":true},` + + `"metadata":{"enable_thinking":"true"}}` + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(*captured).ToNot(BeNil()) + Expect((*captured).RequestMetadata).To(HaveKeyWithValue("enable_thinking", "true")) + Expect((*captured).RequestMetadata).To(HaveKeyWithValue("preserve_thinking", "true")) + }) }) // A model config may default to decayed_mean pooling with a half-life diff --git a/core/schema/openai.go b/core/schema/openai.go index 62b2452621..2df9f9e8d9 100644 --- a/core/schema/openai.go +++ b/core/schema/openai.go @@ -263,6 +263,12 @@ type OpenAIRequest struct { ReasoningEffort string `json:"reasoning_effort,omitempty" yaml:"reasoning_effort"` Metadata map[string]string `json:"metadata,omitempty" yaml:"metadata"` + + // ChatTemplateKwargs is the per-request chat_template_kwargs field that + // llama.cpp's llama-server, vLLM and SGLang accept (e.g. + // {"enable_thinking": false}). Scalar values are folded into Metadata, so + // they take the same path to the backend; an explicit Metadata key wins. + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty" yaml:"chat_template_kwargs"` } type ModelsDataResponse struct { diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 62153f3438..98e92d7d9f 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -751,6 +751,19 @@ and (for `enable_thinking` / `reasoning_effort`) takes effect across every backe reads them, not just llama.cpp. Typed (non-boolean) values are only supported through the model YAML `chat_template_kwargs`, where YAML preserves the type. +Clients written for llama.cpp's `llama-server`, vLLM or SGLang send the same switches as a +top-level `chat_template_kwargs` object. LocalAI accepts that too and folds its scalar values +into `metadata` (booleans become `"true"` / `"false"`, numbers become strings; nested objects +and arrays are ignored). If a key appears in both, `metadata` wins: + +```json +{ + "model": "qwen3", + "messages": [{"role": "user", "content": "hi"}], + "chat_template_kwargs": { "enable_thinking": false } +} +``` + ### Multimodal Backend Options | Option | Type | Default | Description | @@ -1134,8 +1147,9 @@ curl http://localhost:8080/v1/chat/completions \ **Priority order:** 1. Request-level `metadata.enable_thinking` (highest priority) -2. Model config `reasoning.disable` (fallback) -3. Auto-detected from model template (default) +2. Request-level `chat_template_kwargs.enable_thinking` (the llama-server / vLLM form) +3. Model config `reasoning.disable` (fallback) +4. Auto-detected from model template (default) ## Pipeline Configuration