From 40dae953f4f79561de47763f50cef2d976715cae Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:45:43 +0200 Subject: [PATCH] feat: interleaved thinking with tool calls (reasoning_content alias + Anthropic thinking blocks) (#10744) * feat(schema): accept reasoning_content as inbound alias for reasoning Interleaved-thinking clients (cogito, vLLM/DeepSeek-style) emit reasoning_content on assistant turns. Accept it as an inbound alias so reasoning survives the tool-result loop; canonical reasoning wins when both are present. Emission is unchanged (still reasoning). Signed-off-by: Ettore Di Giacinto * test(schema): pin interleaved reasoning+tool_calls round-trip Signed-off-by: Ettore Di Giacinto * test(openai): pin reachedTokenBudget truncation detection Signed-off-by: Ettore Di Giacinto * feat(anthropic): add thinking and signature fields to content blocks Signed-off-by: Ettore Di Giacinto * feat(anthropic): parse inbound thinking blocks into reasoning Signed-off-by: Ettore Di Giacinto * feat(anthropic): emit thinking blocks with synthetic signature on tool turns Extract buildAnthropicContentBlocks so non-streaming content assembly is unit-testable, and prepend a thinking block (with an opaque synthetic signature) before text/tool_use blocks when the request opts into thinking. Signed-off-by: Ettore Di Giacinto * feat(anthropic): stream thinking_delta and signature_delta before tool_use Extract anthropicStreamSequence so the streaming block order is unit-testable, and emit content_block_start(thinking) -> thinking_delta -> signature_delta -> content_block_stop before the tool_use block sequence when thinking is enabled. Signed-off-by: Ettore Di Giacinto * docs: add interleaved thinking with tool calls guide Add a features guide describing interleaved thinking: an assistant turn carrying reasoning and tool_calls together, the reasoning-round-trip contract (including the reasoning_content inbound alias and Anthropic thinking blocks with a synthetic signature), per-backend enablement (reasoning_format for llama.cpp, reasoning_parser/tool_call_parser for vLLM/SGLang plus the vLLM auto-config hook), a worked request/response example, and known limitations. Cross-link from model-configuration, text-generation, and openai-functions. Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/http/endpoints/anthropic/messages.go | 274 +++++++++++++----- .../http/endpoints/anthropic/messages_test.go | 112 +++++++ core/http/endpoints/openai/inference_test.go | 15 + core/schema/anthropic.go | 15 + core/schema/anthropic_test.go | 11 + core/schema/message.go | 19 ++ core/schema/message_test.go | 59 ++++ docs/content/advanced/model-configuration.md | 2 +- docs/content/features/interleaved-thinking.md | 152 ++++++++++ docs/content/features/openai-functions.md | 2 +- docs/content/features/text-generation.md | 2 + 11 files changed, 591 insertions(+), 72 deletions(-) create mode 100644 core/http/endpoints/anthropic/messages_test.go create mode 100644 docs/content/features/interleaved-thinking.md diff --git a/core/http/endpoints/anthropic/messages.go b/core/http/endpoints/anthropic/messages.go index dbdbe9f32..9310cabd9 100644 --- a/core/http/endpoints/anthropic/messages.go +++ b/core/http/endpoints/anthropic/messages.go @@ -3,6 +3,7 @@ package anthropic import ( "encoding/json" "fmt" + "strings" "sync" "time" @@ -244,63 +245,59 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic } } - // No MCP tools to execute, build and return response + // No MCP tools to execute, build and return response. + // Reasoning surfaces as a thinking block only when the client opted in, + // matching Anthropic's extended-thinking gating. + reasoning := functions.ReasoningFromChatDeltas(chatDeltas) + thinkingEnabled := input.Thinking != nil && input.Thinking.Type == "enabled" + var contentBlocks []schema.AnthropicContentBlock var stopReason string if shouldUseFn && len(toolCalls) > 0 { stopReason = "tool_use" - for _, tc := range toolCalls { - var inputArgs map[string]any - if err := json.Unmarshal([]byte(tc.Arguments), &inputArgs); err != nil { - xlog.Warn("Failed to parse tool call arguments as JSON", "error", err, "args", tc.Arguments) - inputArgs = map[string]any{"raw": tc.Arguments} - } - contentBlocks = append(contentBlocks, schema.AnthropicContentBlock{ - Type: "tool_use", - ID: fmt.Sprintf("toolu_%s_%d", id, len(contentBlocks)), - Name: tc.Name, - Input: inputArgs, - }) - } - textContent := functions.ParseTextContent(result, cfg.FunctionsConfig) - if textContent != "" { - contentBlocks = append([]schema.AnthropicContentBlock{{Type: "text", Text: textContent}}, contentBlocks...) - } + contentBlocks = buildAnthropicContentBlocks(buildParams{ + reasoning: reasoning, + thinkingEnabled: thinkingEnabled, + text: functions.ParseTextContent(result, cfg.FunctionsConfig), + toolCalls: funcResultsToToolCalls(toolCalls), + id: id, + }) } else if !shouldUseFn && cfg.FunctionsConfig.AutomaticToolParsingFallback && result != "" { // Automatic tool parsing fallback: no tools in request but model emitted tool call markup parsed := functions.ParseFunctionCall(result, cfg.FunctionsConfig) if len(parsed) > 0 { stopReason = "tool_use" - stripped := functions.StripToolCallMarkup(result) - if stripped != "" { - contentBlocks = append(contentBlocks, schema.AnthropicContentBlock{Type: "text", Text: stripped}) - } - for i, fc := range parsed { - var inputArgs map[string]any - if err := json.Unmarshal([]byte(fc.Arguments), &inputArgs); err != nil { - inputArgs = map[string]any{"raw": fc.Arguments} - } - toolCallID := fc.ID - if toolCallID == "" { - toolCallID = fmt.Sprintf("toolu_%s_%d", id, i) - } - contentBlocks = append(contentBlocks, schema.AnthropicContentBlock{ - Type: "tool_use", - ID: toolCallID, - Name: fc.Name, - Input: inputArgs, - }) - } + contentBlocks = buildAnthropicContentBlocks(buildParams{ + reasoning: reasoning, + thinkingEnabled: thinkingEnabled, + text: functions.StripToolCallMarkup(result), + toolCalls: funcResultsToToolCalls(parsed), + id: id, + }) } else { stopReason = "end_turn" - contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}} + contentBlocks = buildAnthropicContentBlocks(buildParams{ + reasoning: reasoning, + thinkingEnabled: thinkingEnabled, + text: result, + id: id, + }) } } else { stopReason = "end_turn" - contentBlocks = []schema.AnthropicContentBlock{ - {Type: "text", Text: result}, - } + contentBlocks = buildAnthropicContentBlocks(buildParams{ + reasoning: reasoning, + thinkingEnabled: thinkingEnabled, + text: result, + id: id, + }) + } + + // Anthropic responses must carry at least one content block; keep the + // empty-text fallback the pre-refactor assembly guaranteed. + if len(contentBlocks) == 0 { + contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}} } resp := &schema.AnthropicResponse{ @@ -356,6 +353,9 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq } hasMCPTools := mcpExecutor != nil && mcpExecutor.HasTools() + // Reasoning streams as a thinking block only when the client opted in. + thinkingEnabled := input.Thinking != nil && input.Thinking.Type == "enabled" + for mcpIteration := 0; mcpIteration <= mcpMaxIterations; mcpIteration++ { // Re-template on MCP iterations if mcpIteration > 0 { @@ -510,7 +510,9 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq }) } - // Emit tool_use blocks from ChatDeltas + // Emit tool_use blocks from ChatDeltas, preceded by a fully-closed + // thinking block when reasoning is present (Anthropic ordering: + // thinking is streamed and closed before any tool_use starts). if len(deltaToolCalls) > 0 && len(collectedToolCalls) == 0 { collectedToolCalls = deltaToolCalls @@ -522,35 +524,23 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq currentBlockIndex++ inToolCall = true } - for i, tc := range deltaToolCalls { - toolCallID := tc.ID - if toolCallID == "" { - toolCallID = fmt.Sprintf("toolu_%s_%d", id, i) + + seq := anthropicStreamSequence(streamInput{ + reasoningDeltas: []string{functions.ReasoningFromChatDeltas(chatDeltas)}, + thinkingEnabled: thinkingEnabled, + toolCalls: funcResultsToToolCalls(deltaToolCalls), + startIndex: currentBlockIndex, + id: id, + }) + for _, ev := range seq { + sendAnthropicSSE(c, ev) + // Each content block ends with exactly one content_block_stop, + // so advance the running index in lockstep with the sequencer. + if ev.Type == "content_block_stop" { + currentBlockIndex++ } - sendAnthropicSSE(c, schema.AnthropicStreamEvent{ - Type: "content_block_start", - Index: intPtr(currentBlockIndex), - ContentBlock: &schema.AnthropicContentBlock{ - Type: "tool_use", - ID: toolCallID, - Name: tc.Name, - }, - }) - sendAnthropicSSE(c, schema.AnthropicStreamEvent{ - Type: "content_block_delta", - Index: intPtr(currentBlockIndex), - Delta: &schema.AnthropicStreamDelta{ - Type: "input_json_delta", - PartialJSON: tc.Arguments, - }, - }) - sendAnthropicSSE(c, schema.AnthropicStreamEvent{ - Type: "content_block_stop", - Index: intPtr(currentBlockIndex), - }) - currentBlockIndex++ - toolCallsEmitted++ } + toolCallsEmitted += len(deltaToolCalls) } } @@ -697,6 +687,139 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq return nil } +// buildParams carries the pieces of an assistant turn that become Anthropic +// content blocks. It exists so the block-assembly logic is unit-testable in +// isolation from the HTTP handler. +type buildParams struct { + reasoning string + thinkingEnabled bool + text string + toolCalls []schema.ToolCall + id string +} + +// syntheticThinkingSignature returns an opaque, non-cryptographic signature so +// Anthropic SDK clients round-trip the thinking block. Local models have no +// real signature; this marker is accepted-but-ignored on the way back in. +func syntheticThinkingSignature() string { return "localai" } + +// buildAnthropicContentBlocks assembles the content blocks for a non-streaming +// assistant turn. When thinking is enabled and reasoning is present, a thinking +// block is prepended before text and tool_use blocks (Anthropic ordering). +func buildAnthropicContentBlocks(p buildParams) []schema.AnthropicContentBlock { + var blocks []schema.AnthropicContentBlock + if p.thinkingEnabled && p.reasoning != "" { + blocks = append(blocks, schema.AnthropicContentBlock{ + Type: "thinking", + Thinking: p.reasoning, + Signature: syntheticThinkingSignature(), + }) + } + if p.text != "" { + blocks = append(blocks, schema.AnthropicContentBlock{Type: "text", Text: p.text}) + } + for i, tc := range p.toolCalls { + var inputArgs map[string]any + if err := json.Unmarshal([]byte(tc.FunctionCall.Arguments), &inputArgs); err != nil { + inputArgs = map[string]any{"raw": tc.FunctionCall.Arguments} + } + id := tc.ID + if id == "" { + id = fmt.Sprintf("toolu_%s_%d", p.id, i) + } + blocks = append(blocks, schema.AnthropicContentBlock{ + Type: "tool_use", ID: id, Name: tc.FunctionCall.Name, Input: inputArgs, + }) + } + return blocks +} + +// funcResultsToToolCalls adapts the parser's FuncCallResults into schema +// ToolCalls so the pure block builders operate on one tool-call shape. +func funcResultsToToolCalls(results []functions.FuncCallResults) []schema.ToolCall { + out := make([]schema.ToolCall, len(results)) + for i, r := range results { + out[i] = schema.ToolCall{ + ID: r.ID, + Type: "function", + FunctionCall: schema.FunctionCall{Name: r.Name, Arguments: r.Arguments}, + } + } + return out +} + +// streamInput carries the reasoning deltas and tool calls for one streaming +// assistant turn. startIndex and id let the pure sequencer be wired into the +// live loop, which already owns a running block index. +type streamInput struct { + reasoningDeltas []string + thinkingEnabled bool + toolCalls []schema.ToolCall + startIndex int + id string +} + +// anthropicStreamSequence produces the ordered stream events for a thinking +// block followed by tool_use blocks. Per the Anthropic spec the thinking block +// is fully streamed and closed (content_block_stop) before any tool_use block +// starts. Kept pure so ordering is unit-testable without an HTTP stream. +func anthropicStreamSequence(in streamInput) []schema.AnthropicStreamEvent { + var events []schema.AnthropicStreamEvent + idx := in.startIndex + + if in.thinkingEnabled && strings.Join(in.reasoningDeltas, "") != "" { + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_start", + Index: intPtr(idx), + ContentBlock: &schema.AnthropicContentBlock{Type: "thinking"}, + }) + for _, d := range in.reasoningDeltas { + if d == "" { + continue + } + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_delta", + Index: intPtr(idx), + Delta: &schema.AnthropicStreamDelta{Type: "thinking_delta", Thinking: d}, + }) + } + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_delta", + Index: intPtr(idx), + Delta: &schema.AnthropicStreamDelta{Type: "signature_delta", Signature: syntheticThinkingSignature()}, + }) + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_stop", + Index: intPtr(idx), + }) + idx++ + } + + for i, tc := range in.toolCalls { + toolID := tc.ID + if toolID == "" { + toolID = fmt.Sprintf("toolu_%s_%d", in.id, i) + } + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_start", + Index: intPtr(idx), + ContentBlock: &schema.AnthropicContentBlock{Type: "tool_use", ID: toolID, Name: tc.FunctionCall.Name}, + }) + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_delta", + Index: intPtr(idx), + Delta: &schema.AnthropicStreamDelta{Type: "input_json_delta", PartialJSON: tc.FunctionCall.Arguments}, + }) + events = append(events, schema.AnthropicStreamEvent{ + Type: "content_block_stop", + Index: intPtr(idx), + }) + idx++ + } + + return events +} + func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool { tools := make([]functions.Tool, len(funcs)) for i, f := range funcs { @@ -767,6 +890,17 @@ func convertAnthropicToOpenAIMessages(input *schema.AnthropicRequest) []schema.M if text, ok := blockMap["text"].(string); ok { textContent += text } + case "thinking": + // Anthropic interleaved thinking: preserve the model's reasoning so it + // survives the tool-result loop. Signature is Anthropic-cloud specific; + // for local models we read but do not validate it. + if thinking, ok := blockMap["thinking"].(string); ok && thinking != "" { + combined := thinking + if openAIMsg.Reasoning != nil { + combined = *openAIMsg.Reasoning + thinking + } + openAIMsg.Reasoning = &combined + } case "image": // Handle image content if source, ok := blockMap["source"].(map[string]any); ok { diff --git a/core/http/endpoints/anthropic/messages_test.go b/core/http/endpoints/anthropic/messages_test.go new file mode 100644 index 000000000..4db880f19 --- /dev/null +++ b/core/http/endpoints/anthropic/messages_test.go @@ -0,0 +1,112 @@ +package anthropic + +import ( + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Anthropic thinking inbound", func() { + It("maps an assistant thinking block to Message.Reasoning", func() { + req := &schema.AnthropicRequest{ + Messages: []schema.AnthropicMessage{ + {Role: "assistant", Content: []any{ + map[string]any{"type": "thinking", "thinking": "I should call get_weather", "signature": "sig_1"}, + map[string]any{"type": "text", "text": "Checking."}, + }}, + }, + } + msgs := convertAnthropicToOpenAIMessages(req) + Expect(msgs).To(HaveLen(1)) + Expect(msgs[0].Reasoning).NotTo(BeNil()) + Expect(*msgs[0].Reasoning).To(Equal("I should call get_weather")) + }) +}) + +var _ = Describe("Anthropic thinking outbound (non-stream)", func() { + It("emits a thinking block before tool_use when reasoning is present", func() { + blocks := buildAnthropicContentBlocks(buildParams{ + reasoning: "I need the weather", + thinkingEnabled: true, + text: "", + toolCalls: []schema.ToolCall{{ID: "call_1", Type: "function", + FunctionCall: schema.FunctionCall{Name: "get_weather", Arguments: `{"city":"Rome"}`}}}, + id: "abc", + }) + Expect(blocks[0].Type).To(Equal("thinking")) + Expect(blocks[0].Thinking).To(Equal("I need the weather")) + Expect(blocks[0].Signature).NotTo(BeEmpty()) + Expect(blocks[1].Type).To(Equal("tool_use")) + }) + + It("omits the thinking block when thinking is not enabled", func() { + blocks := buildAnthropicContentBlocks(buildParams{ + reasoning: "hidden", thinkingEnabled: false, text: "hi", id: "abc", + }) + for _, b := range blocks { + Expect(b.Type).NotTo(Equal("thinking")) + } + }) +}) + +var _ = Describe("Anthropic thinking outbound (stream)", func() { + It("sequences a thinking block before tool_use in streaming order", func() { + seq := anthropicStreamSequence(streamInput{ + reasoningDeltas: []string{"think ", "more"}, + thinkingEnabled: true, + toolCalls: []schema.ToolCall{{ID: "call_1", Type: "function", + FunctionCall: schema.FunctionCall{Name: "f", Arguments: "{}"}}}, + }) + types := eventTypes(seq) + Expect(types).To(ContainElements( + "content_block_start", "thinking_delta", "signature_delta", "content_block_stop")) + Expect(indexOf(types, "content_block_stop")).To(BeNumerically("<", indexOf(types, "tool_use_start"))) + }) + + It("omits the thinking sequence when thinking is not enabled", func() { + seq := anthropicStreamSequence(streamInput{ + reasoningDeltas: []string{"hidden"}, + thinkingEnabled: false, + toolCalls: []schema.ToolCall{{ID: "call_1", Type: "function", + FunctionCall: schema.FunctionCall{Name: "f", Arguments: "{}"}}}, + }) + types := eventTypes(seq) + Expect(types).NotTo(ContainElement("thinking_delta")) + Expect(types).NotTo(ContainElement("signature_delta")) + }) +}) + +// eventTypes maps each streaming event to a stable logical label so ordering +// assertions read naturally: content_block_start of a tool_use block is +// surfaced as "tool_use_start", and delta events surface their delta type. +func eventTypes(events []schema.AnthropicStreamEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + switch e.Type { + case "content_block_start": + if e.ContentBlock != nil && e.ContentBlock.Type == "tool_use" { + out = append(out, "tool_use_start") + continue + } + out = append(out, e.Type) + case "content_block_delta": + if e.Delta != nil { + out = append(out, e.Delta.Type) + continue + } + out = append(out, e.Type) + default: + out = append(out, e.Type) + } + } + return out +} + +func indexOf(items []string, target string) int { + for i, s := range items { + if s == target { + return i + } + } + return -1 +} diff --git a/core/http/endpoints/openai/inference_test.go b/core/http/endpoints/openai/inference_test.go index 7debd15fc..db7d3ff91 100644 --- a/core/http/endpoints/openai/inference_test.go +++ b/core/http/endpoints/openai/inference_test.go @@ -407,6 +407,21 @@ var _ = Describe("ComputeChoices", func() { Expect(reachedTokenBudget(100, ptr(100))).To(BeTrue()) Expect(reachedTokenBudget(101, ptr(100))).To(BeTrue()) }) + + // Truncation labeling for agentic clients (cogito): a reasoning model + // that meets its output ceiling must report finish_reason=length so the + // caller can detect the cutoff rather than treat it as a clean stop. + It("reports the budget as reached when completion meets max_tokens", func() { + max := 16 + Expect(reachedTokenBudget(16, &max)).To(BeTrue()) + Expect(reachedTokenBudget(15, &max)).To(BeFalse()) + }) + + It("does not flag a budget when max_tokens is zero or nil", func() { + zero := 0 + Expect(reachedTokenBudget(100, &zero)).To(BeFalse()) + Expect(reachedTokenBudget(100, nil)).To(BeFalse()) + }) }) Context("max_tokens budget exhausted on reasoning (issue #9716)", func() { diff --git a/core/schema/anthropic.go b/core/schema/anthropic.go index 182dbfc36..1d55063d8 100644 --- a/core/schema/anthropic.go +++ b/core/schema/anthropic.go @@ -49,11 +49,22 @@ type AnthropicRequest struct { Tools []AnthropicTool `json:"tools,omitempty"` ToolChoice any `json:"tool_choice,omitempty"` + // Thinking gates extended-thinking output. We only surface thinking + // content blocks when the client explicitly opts in, matching Anthropic's + // API where thinking is off unless requested. + Thinking *AnthropicThinkingParam `json:"thinking,omitempty"` + // Internal fields for request handling Context context.Context `json:"-"` Cancel context.CancelFunc `json:"-"` } +// AnthropicThinkingParam is the request-side extended-thinking toggle. +type AnthropicThinkingParam struct { + Type string `json:"type"` // "enabled" | "disabled" + BudgetTokens int `json:"budget_tokens,omitempty"` +} + // ModelName implements the LocalAIRequest interface func (ar *AnthropicRequest) ModelName(s *string) string { if s != nil { @@ -86,6 +97,8 @@ type AnthropicContentBlock struct { ToolUseID string `json:"tool_use_id,omitempty"` Content any `json:"content,omitempty"` IsError *bool `json:"is_error,omitempty"` + Thinking string `json:"thinking,omitempty"` + Signature string `json:"signature,omitempty"` } // AnthropicImageSource represents an image source in Anthropic format @@ -131,6 +144,8 @@ type AnthropicStreamDelta struct { PartialJSON string `json:"partial_json,omitempty"` StopReason *string `json:"stop_reason,omitempty"` StopSequence *string `json:"stop_sequence,omitempty"` + Thinking string `json:"thinking,omitempty"` + Signature string `json:"signature,omitempty"` } // AnthropicStreamMessage represents the message object in streaming events diff --git a/core/schema/anthropic_test.go b/core/schema/anthropic_test.go index f21e5e44f..386fb8fa2 100644 --- a/core/schema/anthropic_test.go +++ b/core/schema/anthropic_test.go @@ -190,6 +190,17 @@ var _ = Describe("Anthropic Schema", func() { }) }) + Describe("AnthropicContentBlock", func() { + It("round-trips a thinking content block with signature", func() { + raw := `{"type":"thinking","thinking":"step by step","signature":"sig_abc"}` + var b schema.AnthropicContentBlock + Expect(json.Unmarshal([]byte(raw), &b)).To(Succeed()) + Expect(b.Type).To(Equal("thinking")) + Expect(b.Thinking).To(Equal("step by step")) + Expect(b.Signature).To(Equal("sig_abc")) + }) + }) + Describe("AnthropicErrorResponse", func() { It("should marshal an error response", func() { resp := schema.AnthropicErrorResponse{ diff --git a/core/schema/message.go b/core/schema/message.go index d55e91345..27e5b72aa 100644 --- a/core/schema/message.go +++ b/core/schema/message.go @@ -38,6 +38,25 @@ type Message struct { Reasoning *string `json:"reasoning,omitempty" yaml:"reasoning,omitempty"` } +// UnmarshalJSON decodes Message, accepting reasoning_content as an inbound +// alias for reasoning so vLLM/DeepSeek/OpenAI-SDK style clients that emit +// reasoning_content on assistant turns round-trip through the interleaved +// thinking loop. Canonical reasoning wins when both are present. +func (m *Message) UnmarshalJSON(data []byte) error { + type messageAlias Message + aux := struct { + *messageAlias + ReasoningContent *string `json:"reasoning_content,omitempty"` + }{messageAlias: (*messageAlias)(m)} + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + if m.Reasoning == nil && aux.ReasoningContent != nil { + m.Reasoning = aux.ReasoningContent + } + return nil +} + type ToolCall struct { Index int `json:"index"` ID string `json:"id"` diff --git a/core/schema/message_test.go b/core/schema/message_test.go index eb471b57b..0828b8fc0 100644 --- a/core/schema/message_test.go +++ b/core/schema/message_test.go @@ -394,5 +394,64 @@ var _ = Describe("LLM tests", func() { Expect(protoMessages).To(HaveLen(1)) Expect(protoMessages[0].Content).To(Equal("Hello")) }) + + // Interleaved thinking: an assistant turn can emit a thinking block and + // a tool call in the same message. ToProto sets ReasoningContent and + // ToolCalls in independent branches, so both must survive the round-trip. + It("carries reasoning AND tool_calls together on one assistant message", func() { + reasoning := "let me check the weather first" + messages := Messages{ + { + Role: "assistant", + Content: "", + Reasoning: &reasoning, + ToolCalls: []ToolCall{ + {Index: 0, ID: "call_1", Type: "function", + FunctionCall: FunctionCall{Name: "get_weather", Arguments: `{"city":"Rome"}`}}, + }, + }, + } + proto := messages.ToProto() + Expect(proto).To(HaveLen(1)) + Expect(proto[0].ReasoningContent).To(Equal("let me check the weather first")) + Expect(proto[0].ToolCalls).NotTo(BeEmpty()) + }) + + // Multi-turn continuity: when the interleaved assistant turn is replayed + // after its tool result, reasoning_content and the tool linkage must both + // persist so the model sees a coherent conversation on the next round. + It("preserves reasoning on a replayed assistant turn across a tool result", func() { + r1 := "reason before call" + messages := Messages{ + {Role: "user", Content: "weather in Rome?"}, + {Role: "assistant", Content: "", Reasoning: &r1, + ToolCalls: []ToolCall{{Index: 0, ID: "call_1", Type: "function", + FunctionCall: FunctionCall{Name: "get_weather", Arguments: `{"city":"Rome"}`}}}}, + {Role: "tool", Content: "22C sunny", ToolCallID: "call_1"}, + } + proto := messages.ToProto() + Expect(proto).To(HaveLen(3)) + Expect(proto[1].ReasoningContent).To(Equal("reason before call")) + Expect(proto[1].ToolCalls).NotTo(BeEmpty()) + Expect(proto[2].ToolCallId).To(Equal("call_1")) + }) + }) + + Context("reasoning_content inbound alias", func() { + It("decodes reasoning_content as an inbound alias for Reasoning", func() { + var m Message + err := json.Unmarshal([]byte(`{"role":"assistant","reasoning_content":"thinking..."}`), &m) + Expect(err).NotTo(HaveOccurred()) + Expect(m.Reasoning).NotTo(BeNil()) + Expect(*m.Reasoning).To(Equal("thinking...")) + }) + + It("prefers reasoning over reasoning_content when both are present", func() { + var m Message + err := json.Unmarshal([]byte(`{"role":"assistant","reasoning":"canonical","reasoning_content":"alias"}`), &m) + Expect(err).NotTo(HaveOccurred()) + Expect(m.Reasoning).NotTo(BeNil()) + Expect(*m.Reasoning).To(Equal("canonical")) + }) }) }) diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index f71cf3d0d..2fcb84c35 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -403,7 +403,7 @@ Pre-converted GGUFs with MTP heads are published on the [ggml-org HuggingFace or ### Reasoning Models (DeepSeek-R1, Qwen3, etc.) -These load-time options control how the backend parses `` reasoning blocks and how much budget the model is allowed for thinking. They are set per model via the `options:` array. +These load-time options control how the backend parses `` reasoning blocks and how much budget the model is allowed for thinking. They are set per model via the `options:` array. For how reasoning is returned alongside tool calls and survives the tool-result round trip, see [Interleaved Thinking with Tool Calls]({{%relref "features/interleaved-thinking" %}}). | Option | Type | Default | Description | |--------|------|---------|-------------| diff --git a/docs/content/features/interleaved-thinking.md b/docs/content/features/interleaved-thinking.md new file mode 100644 index 000000000..1104e2c11 --- /dev/null +++ b/docs/content/features/interleaved-thinking.md @@ -0,0 +1,152 @@ + ++++ +disableToc = false +title = "Interleaved Thinking with Tool Calls" +weight = 17 +url = "/features/interleaved-thinking/" ++++ + +Reasoning models can "think" before they answer. When such a model also calls a tool, the useful behaviour is for the thinking and the tool call to travel together, and for that thinking to survive the tool-result round trip. LocalAI calls this **interleaved thinking**: a single assistant turn carries both `reasoning` and `tool_calls`, and the client hands the `reasoning` back on the next turn so the model's chain of thought is not lost when the tool result is appended. + +This matters because a tool-calling loop is multi-turn. The model reasons, asks for a tool, your client runs the tool, and then you call the model again with the tool result. Without interleaved thinking the reasoning produced in the first turn is discarded, and the model has to reconstruct its plan from scratch. With it, the reasoning is echoed back and the model continues where it left off. + +See also: [OpenAI Functions and Tools]({{%relref "features/openai-functions" %}}) for how tool calls are extracted per backend, [Text Generation (GPT)]({{%relref "features/text-generation" %}}) for the chat completions basics, and [Advanced model configuration]({{%relref "advanced/model-configuration" %}}) for the model `options` referenced below. + +## The round-trip contract + +An assistant turn that both reasons and calls a tool returns the two fields side by side: + +- `reasoning` holds the model's thinking. +- `tool_calls` holds the structured calls. +- `finish_reason` is `tool_calls`. + +Your client runs the tool, then sends the conversation back with: + +1. the original assistant message (including its `reasoning` and `tool_calls`), and +2. a `tool` role message carrying the tool result. + +LocalAI reads the returned `reasoning` back into the model's context so the chain is continuous. + +### Field naming + +**OpenAI chat completions** (`/v1/chat/completions`): the response carries `reasoning` alongside `tool_calls`. On inbound assistant messages LocalAI now also accepts `reasoning_content` as an alias for `reasoning`. This alias exists because several clients (vLLM, DeepSeek, and cogito) emit the field under the name `reasoning_content`; either name is accepted and mapped to the same internal field. + +**Anthropic Messages** (`/v1/messages`): reasoning is carried as `thinking` content blocks. On the local path LocalAI emits a `thinking` block before the `tool_use` block, and reads inbound `thinking` blocks back into reasoning. Because local models produce no cryptographic signature, LocalAI attaches a synthetic opaque `signature` to the emitted block, and does not validate the signature on inbound blocks. The `thinking` block is only emitted when the request opts in with the `thinking` parameter: + +```json +{ + "model": "your-reasoning-model", + "max_tokens": 1024, + "thinking": { "type": "enabled" }, + "messages": [ + { "role": "user", "content": "What is the weather in Rome?" } + ] +} +``` + +## Enabling reasoning per backend + +Interleaved thinking requires the backend to separate the model's thinking from its final answer. How you turn that on depends on the backend. + +### llama.cpp + +Set the `reasoning_format` model option. Accepted values: `none`, `auto`, `deepseek`, `deepseek-legacy`. `auto` lets the backend pick based on the model's chat template; `deepseek` and `deepseek-legacy` force the DeepSeek-style `...` extraction. + +```yaml +name: my-reasoning-model +backend: llama-cpp +parameters: + model: my-reasoning-model.gguf +options: + - reasoning_format:auto +``` + +### vLLM + +Set the `reasoning_parser` model option to vLLM's native reasoning parser for the model family. LocalAI also ships an auto-configuration hook (`core/config/hooks_vllm.go`) that sets the reasoning parser and the tool-call parser together for known families, so for gallery-imported vLLM models this is often configured for you. + +```yaml +name: my-vllm-model +backend: vllm +options: + - reasoning_parser:deepseek_r1 + - tool_parser:hermes +``` + +### SGLang + +Set both the `reasoning_parser` and the `tool_call_parser` model options. + +```yaml +name: my-sglang-model +backend: sglang +options: + - reasoning_parser:deepseek-r1 + - tool_call_parser:qwen25 +``` + +## Worked example + +Request a chat completion with a tool defined and a prompt that requires the model to reason before acting: + +```bash +curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ + "model": "your-reasoning-model", + "messages": [ + { "role": "user", "content": "What is the weather in Rome?" } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": { "type": "string" } + }, + "required": ["city"] + } + } + } + ] +}' +``` + +The response message carries both the reasoning and the tool call, and `finish_reason` is `tool_calls`: + +```json +{ + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "", + "reasoning": "Okay, the user is asking about the weather in Rome... I need to call get_weather with city Rome.", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\":\"Rome\"}" + } + } + ] + } + } + ] +} +``` + +To continue, run `get_weather`, then send the conversation back with the assistant message (keeping its `reasoning` and `tool_calls`) followed by a `tool` message holding the result. LocalAI feeds the returned reasoning back into context so the model resumes its chain of thought. + +## Known limitations + +**Streaming Anthropic `thinking` blocks.** In streaming mode, a `thinking` block is currently emitted only on the tool-call path that goes through the llama.cpp C++ autoparser. A plain streaming text-only turn, or a tool turn resolved through the inline token path, does not stream a `thinking` block. The equivalent non-streaming request does return one. Non-streaming emits thinking on all branches; only the streaming path has this gap. + +**Upstream llama.cpp leak on newest hybrid models.** On the newest hybrid reasoning models (Qwen3.5 / Qwen3.6) there is an upstream llama.cpp bug where tool calls can leak into `reasoning_content` instead of being parsed into `tool_calls`. This is tracked upstream at [ggml-org/llama.cpp discussion #23351](https://github.com/ggml-org/llama.cpp/discussions/23351). + +**Reasoning budget vs the tool call.** If a reasoning model's thinking exhausts the output budget (`max_tokens`) before it emits the tool call, no tool call is produced. Give the model enough `max_tokens` to cover both the reasoning and the call. In this case LocalAI reports `finish_reason: "length"`. diff --git a/docs/content/features/openai-functions.md b/docs/content/features/openai-functions.md index 4893cd4ef..18019cf52 100644 --- a/docs/content/features/openai-functions.md +++ b/docs/content/features/openai-functions.md @@ -28,7 +28,7 @@ LocalAI also supports [JSON mode](https://platform.openai.com/docs/guides/text-g | `mlx` | `mlx_lm.tool_parsers` — **auto-detected from the chat template**, no configuration needed | | `mlx-vlm` | `mlx_vlm.tool_parsers` (with fallback to mlx-lm parsers) — **auto-detected from the chat template**, no configuration needed | -Reasoning content (`...` blocks from DeepSeek R1, Qwen3, Gemma 4, etc.) is returned in the OpenAI `reasoning_content` field on the same backends. +Reasoning content (`...` blocks from DeepSeek R1, Qwen3, Gemma 4, etc.) is returned in the OpenAI `reasoning_content` field on the same backends. When a model both reasons and calls a tool in the same turn, see [Interleaved Thinking with Tool Calls]({{%relref "features/interleaved-thinking" %}}) for how the reasoning survives the tool-result round trip. ## Setup diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index 3d45ec6b9..ef62695e5 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -31,6 +31,8 @@ curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/jso Available additional parameters: `top_p`, `top_k`, `max_tokens` +Reasoning models return their thinking in the `reasoning` field. When a model reasons and calls a tool in the same turn, see [Interleaved Thinking with Tool Calls]({{%relref "features/interleaved-thinking" %}}). + ### Edit completions https://platform.openai.com/docs/api-reference/edits