diff --git a/core/http/endpoints/anthropic/messages.go b/core/http/endpoints/anthropic/messages.go index f7a09b4c95..3990916811 100644 --- a/core/http/endpoints/anthropic/messages.go +++ b/core/http/endpoints/anthropic/messages.go @@ -310,6 +310,7 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic if len(contentBlocks) == 0 { contentBlocks = []schema.AnthropicContentBlock{{Type: "text", Text: result}} } + stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens) resp := &schema.AnthropicResponse{ ID: fmt.Sprintf("msg_%s", id), @@ -671,6 +672,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq if toolCallsEmitted > 0 { stopReason = "tool_use" } + stopReason = budgetStopReason(stopReason, tokenUsage.Completion, input.MaxTokens) sendAnthropicSSE(c, schema.AnthropicStreamEvent{ Type: "message_delta", @@ -831,6 +833,16 @@ func anthropicStreamSequence(in streamInput) []schema.AnthropicStreamEvent { return events } +// budgetStopReason reports "max_tokens" instead of "end_turn" when the +// generation used up the request's max_tokens, mirroring the "length" +// finish_reason of the OpenAI endpoint. tool_use is left as is. +func budgetStopReason(stopReason string, completionTokens, maxTokens int) string { + if stopReason == "end_turn" && maxTokens > 0 && completionTokens >= maxTokens { + return "max_tokens" + } + return stopReason +} + func convertFuncsToOpenAITools(funcs functions.Functions) []functions.Tool { tools := make([]functions.Tool, len(funcs)) for i, f := range funcs { diff --git a/core/http/endpoints/anthropic/messages_test.go b/core/http/endpoints/anthropic/messages_test.go index 160e7ecb07..13a8f7617a 100644 --- a/core/http/endpoints/anthropic/messages_test.go +++ b/core/http/endpoints/anthropic/messages_test.go @@ -122,3 +122,15 @@ var _ = Describe("Typed inbound conversion", func() { Expect(req.Messages[0].Content).To(Equal(blocks)) }) }) + +var _ = Describe("Anthropic stop_reason for an exhausted max_tokens", func() { + DescribeTable("budgetStopReason", + func(stopReason string, completion, maxTokens int, expected string) { + Expect(budgetStopReason(stopReason, completion, maxTokens)).To(Equal(expected)) + }, + Entry("under the limit", "end_turn", 9, 10, "end_turn"), + Entry("at the limit", "end_turn", 10, 10, "max_tokens"), + Entry("tool_use is kept", "tool_use", 10, 10, "tool_use"), + Entry("no limit", "end_turn", 10, 0, "end_turn"), + ) +}) diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index f7cf77be98..fb846e42e4 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -224,6 +224,8 @@ Streaming responses use Server-Sent Events (SSE) format with event types: `messa } ``` +`stop_reason` is `end_turn`, `tool_use`, or `max_tokens` when the reply used up `max_tokens`. + ### Open Responses API LocalAI supports the Open Responses API specification, which provides a standardized interface for AI model interactions with support for background processing, streaming, tool calling, and advanced features like reasoning. diff --git a/tests/e2e/e2e_anthropic_test.go b/tests/e2e/e2e_anthropic_test.go index e13801c02e..542158a714 100644 --- a/tests/e2e/e2e_anthropic_test.go +++ b/tests/e2e/e2e_anthropic_test.go @@ -134,6 +134,56 @@ var _ = Describe("Anthropic API E2E test", func() { }) }) + Context("max_tokens stop reason", func() { + // The mock backend reports 10 completion tokens for a plain + // non-streaming reply and one per character when streaming, so a + // max_tokens of 5 is always exhausted. + It("reports max_tokens when the non-streaming reply hits the limit", func() { + message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{ + Model: "mock-model", + MaxTokens: 5, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")), + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(message.Usage.OutputTokens).To(BeNumerically(">=", 5)) + Expect(string(message.StopReason)).To(Equal("max_tokens")) + }) + + It("reports max_tokens when the streamed reply hits the limit", func() { + stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{ + Model: "mock-model", + MaxTokens: 5, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")), + }, + }) + message := anthropic.Message{} + for stream.Next() { + Expect(message.Accumulate(stream.Current())).To(Succeed()) + } + Expect(stream.Err()).ToNot(HaveOccurred()) + Expect(string(message.StopReason)).To(Equal("max_tokens")) + }) + + It("keeps end_turn when the streamed reply stays under the limit", func() { + stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{ + Model: "mock-model", + MaxTokens: 1024, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("Hello")), + }, + }) + message := anthropic.Message{} + for stream.Next() { + Expect(message.Accumulate(stream.Current())).To(Succeed()) + } + Expect(stream.Err()).ToNot(HaveOccurred()) + Expect(string(message.StopReason)).To(Equal("end_turn")) + }) + }) + Context("Tool calling", func() { It("handles tool calls in non-streaming mode", func() { message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{