fix(anthropic): report max_tokens when the token budget is exhausted

/v1/messages always answered stop_reason "end_turn" for text replies,
even when generation stopped at max_tokens. Clients that continue a
truncated turn (the Anthropic SDKs, Claude Code) treated the cut-off
reply as complete. Use the same check the OpenAI endpoint uses for
finish_reason "length", in both the streaming and non-streaming paths.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: jaeminbird <jaeminbird@gmail.com>
This commit is contained in:
jaeminbird committed 2026-10-29 01:27:47 -04:00
1 parent 85617924fd
commit d905ab5fdc
4 files changed
+76

No files matched your search

+12
View File
@@ -310,6 +310,7 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
if len(contentBlocks) == 0 {
contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}}
}
stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens)
resp := &schema.AnthropicResponse{
ID: fmt.Sprintf("msg_%s", id),
@@ -671,6 +672,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
if toolCallsEmitted > 0 {
stopReason = "tool_use"
}
stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens)
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
Type: "message_delta",
@@ -831,6 +833,16 @@ func anthropicStreamSequence(in streamInput) []schema.AnthropicStreamEvent {
return events
}
// budgetStopReason reports "max_tokens" instead of "end_turn" when the
// generation used up the request's max_tokens, mirroring the "length"
// finish_reason of the OpenAI endpoint. tool_use is left as is.
func budgetStopReason(stopReason string, completionTokens, maxTokens int) string {
if stopReason == "end_turn" && maxTokens > 0 && completionTokens >= maxTokens {
return "max_tokens"
}
return stopReason
}
func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool {
tools := make([]functions.Tool, len(funcs))
for i, f := range funcs {
@@ -122,3 +122,15 @@ var _ = Describe("Typed inbound conversion", func() {
Expect(req.Messages[0].Content).To(Equal(blocks))
})
})
var _ = Describe("Anthropic stop_reason for an exhausted max_tokens", func() {
DescribeTable("budgetStopReason",
func(stopReason string, completion, maxTokens int, expected string) {
Expect(budgetStopReason(stopReason, completion, maxTokens)).To(Equal(expected))
},
Entry("under the limit", "end_turn", 9, 10, "end_turn"),
Entry("at the limit", "end_turn", 10, 10, "max_tokens"),
Entry("tool_use is kept", "tool_use", 10, 10, "tool_use"),
Entry("no limit", "end_turn", 10, 0, "end_turn"),
)
})
+2
View File
@@ -224,6 +224,8 @@ Streaming responses use Server-Sent Events (SSE) format with event types: `messa
}
```
`stop_reason` is `end_turn`, `tool_use`, or `max_tokens` when the reply used up `max_tokens`.
### Open Responses API
LocalAI supports the Open Responses API specification, which provides a standardized interface for AI model interactions with support for background processing, streaming, tool calling, and advanced features like reasoning.
+50
View File
@@ -134,6 +134,56 @@ var _ = Describe("Anthropic API E2E test", func() {
})
})
Context("max_tokens stop reason", func() {
// The mock backend reports 10 completion tokens for a plain
// non-streaming reply and one per character when streaming, so a
// max_tokens of 5 is always exhausted.
It("reports max_tokens when the non-streaming reply hits the limit", func() {
message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{
Model: "mock-model",
MaxTokens: 5,
Messages: []anthropic.MessageParam{
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
},
})
Expect(err).ToNot(HaveOccurred())
Expect(message.Usage.OutputTokens).To(BeNumerically(">=", 5))
Expect(string(message.StopReason)).To(Equal("max_tokens"))
})
It("reports max_tokens when the streamed reply hits the limit", func() {
stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{
Model: "mock-model",
MaxTokens: 5,
Messages: []anthropic.MessageParam{
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
},
})
message := anthropic.Message{}
for stream.Next() {
Expect(message.Accumulate(stream.Current())).To(Succeed())
}
Expect(stream.Err()).ToNot(HaveOccurred())
Expect(string(message.StopReason)).To(Equal("max_tokens"))
})
It("keeps end_turn when the streamed reply stays under the limit", func() {
stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{
Model: "mock-model",
MaxTokens: 1024,
Messages: []anthropic.MessageParam{
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
},
})
message := anthropic.Message{}
for stream.Next() {
Expect(message.Accumulate(stream.Current())).To(Succeed())
}
Expect(stream.Err()).ToNot(HaveOccurred())
Expect(string(message.StopReason)).To(Equal("end_turn"))
})
})
Context("Tool calling", func() {
It("handles tool calls in non-streaming mode", func() {
message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{