mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-06 04:23:15 -05:00
fix(anthropic): report max_tokens when the token budget is exhausted
/v1/messages always answered stop_reason "end_turn" for text replies, even when generation stopped at max_tokens. Clients that continue a truncated turn (the Anthropic SDKs, Claude Code) treated the cut-off reply as complete. Use the same check the OpenAI endpoint uses for finish_reason "length", in both the streaming and non-streaming paths. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: jaeminbird <jaeminbird@gmail.com>
This commit is contained in:
4 files changed
+76
No files matched your search
@@ -310,6 +310,7 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic
|
||||
if len(contentBlocks) == 0 {
|
||||
contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}}
|
||||
}
|
||||
stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens)
|
||||
|
||||
resp := &schema.AnthropicResponse{
|
||||
ID: fmt.Sprintf("msg_%s", id),
|
||||
@@ -671,6 +672,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq
|
||||
if toolCallsEmitted > 0 {
|
||||
stopReason = "tool_use"
|
||||
}
|
||||
stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens)
|
||||
|
||||
sendAnthropicSSE(c, schema.AnthropicStreamEvent{
|
||||
Type: "message_delta",
|
||||
@@ -831,6 +833,16 @@ func anthropicStreamSequence(in streamInput) []schema.AnthropicStreamEvent {
|
||||
return events
|
||||
}
|
||||
|
||||
// budgetStopReason reports "max_tokens" instead of "end_turn" when the
|
||||
// generation used up the request's max_tokens, mirroring the "length"
|
||||
// finish_reason of the OpenAI endpoint. tool_use is left as is.
|
||||
func budgetStopReason(stopReason string, completionTokens, maxTokens int) string {
|
||||
if stopReason == "end_turn" && maxTokens > 0 && completionTokens >= maxTokens {
|
||||
return "max_tokens"
|
||||
}
|
||||
return stopReason
|
||||
}
|
||||
|
||||
func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool {
|
||||
tools := make([]functions.Tool, len(funcs))
|
||||
for i, f := range funcs {
|
||||
|
||||
@@ -122,3 +122,15 @@ var _ = Describe("Typed inbound conversion", func() {
|
||||
Expect(req.Messages[0].Content).To(Equal(blocks))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("Anthropic stop_reason for an exhausted max_tokens", func() {
|
||||
DescribeTable("budgetStopReason",
|
||||
func(stopReason string, completion, maxTokens int, expected string) {
|
||||
Expect(budgetStopReason(stopReason, completion, maxTokens)).To(Equal(expected))
|
||||
},
|
||||
Entry("under the limit", "end_turn", 9, 10, "end_turn"),
|
||||
Entry("at the limit", "end_turn", 10, 10, "max_tokens"),
|
||||
Entry("tool_use is kept", "tool_use", 10, 10, "tool_use"),
|
||||
Entry("no limit", "end_turn", 10, 0, "end_turn"),
|
||||
)
|
||||
})
|
||||
@@ -224,6 +224,8 @@ Streaming responses use Server-Sent Events (SSE) format with event types: `messa
|
||||
}
|
||||
```
|
||||
|
||||
`stop_reason` is `end_turn`, `tool_use`, or `max_tokens` when the reply used up `max_tokens`.
|
||||
|
||||
### Open Responses API
|
||||
|
||||
LocalAI supports the Open Responses API specification, which provides a standardized interface for AI model interactions with support for background processing, streaming, tool calling, and advanced features like reasoning.
|
||||
|
||||
@@ -134,6 +134,56 @@ var _ = Describe("Anthropic API E2E test", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Context("max_tokens stop reason", func() {
|
||||
// The mock backend reports 10 completion tokens for a plain
|
||||
// non-streaming reply and one per character when streaming, so a
|
||||
// max_tokens of 5 is always exhausted.
|
||||
It("reports max_tokens when the non-streaming reply hits the limit", func() {
|
||||
message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{
|
||||
Model: "mock-model",
|
||||
MaxTokens: 5,
|
||||
Messages: []anthropic.MessageParam{
|
||||
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(message.Usage.OutputTokens).To(BeNumerically(">=", 5))
|
||||
Expect(string(message.StopReason)).To(Equal("max_tokens"))
|
||||
})
|
||||
|
||||
It("reports max_tokens when the streamed reply hits the limit", func() {
|
||||
stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{
|
||||
Model: "mock-model",
|
||||
MaxTokens: 5,
|
||||
Messages: []anthropic.MessageParam{
|
||||
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
|
||||
},
|
||||
})
|
||||
message := anthropic.Message{}
|
||||
for stream.Next() {
|
||||
Expect(message.Accumulate(stream.Current())).To(Succeed())
|
||||
}
|
||||
Expect(stream.Err()).ToNot(HaveOccurred())
|
||||
Expect(string(message.StopReason)).To(Equal("max_tokens"))
|
||||
})
|
||||
|
||||
It("keeps end_turn when the streamed reply stays under the limit", func() {
|
||||
stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{
|
||||
Model: "mock-model",
|
||||
MaxTokens: 1024,
|
||||
Messages: []anthropic.MessageParam{
|
||||
anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")),
|
||||
},
|
||||
})
|
||||
message := anthropic.Message{}
|
||||
for stream.Next() {
|
||||
Expect(message.Accumulate(stream.Current())).To(Succeed())
|
||||
}
|
||||
Expect(stream.Err()).ToNot(HaveOccurred())
|
||||
Expect(string(message.StopReason)).To(Equal("end_turn"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("Tool calling", func() {
|
||||
It("handles tool calls in non-streaming mode", func() {
|
||||
message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{
|
||||
|
||||
Reference in new issue
Block a user