From 5bd07865a363db2caacd879ea511b4a6db66b918 Mon Sep 17 00:00:00 2001 From: jaeminbird Date: Wed, 28 Oct 2026 22:09:48 -0400 Subject: [PATCH] fix(anthropic): honor object tool_choice on /v1/messages Anthropic sends tool_choice as an object: {"type": "auto" | "any" | "none"} or {"type": "tool", "name": ...}. The endpoint only read the "tool" object, so {"type": "none"} still offered the tools and {"type": "any"} did not require a call. The object was also forwarded to the backend unchanged, and the llama.cpp backend turns any object tool_choice into "required", so even {"type": "auto"} forced a tool call there. Read the object form, and forward the OpenAI equivalent. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: jaeminbird --- core/http/endpoints/anthropic/messages.go | 65 ++++++++++++------- .../http/endpoints/anthropic/messages_test.go | 37 +++++++++++ docs/content/features/text-generation.md | 4 +- tests/e2e/e2e_anthropic_test.go | 53 +++++++++++++++ tests/e2e/mock-backend/main.go | 10 +++ 5 files changed, 144 insertions(+), 25 deletions(-) diff --git a/core/http/endpoints/anthropic/messages.go b/core/http/endpoints/anthropic/messages.go index f7a09b4c95..cde515f843 100644 --- a/core/http/endpoints/anthropic/messages.go +++ b/core/http/endpoints/anthropic/messages.go @@ -174,7 +174,7 @@ func handleAnthropicNonStream(c echo.Context, id string, input *schema.Anthropic // Populate openAIReq fields for ComputeChoices openAIReq.Tools = convertFuncsToOpenAITools(funcs) - openAIReq.ToolsChoice = input.ToolChoice + openAIReq.ToolsChoice = openAIToolChoice(input.ToolChoice) openAIReq.Metadata = input.Metadata var result string @@ -480,7 +480,7 @@ func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicReq // Populate openAIReq fields for ComputeChoices openAIReq.Tools = convertFuncsToOpenAITools(funcs) - openAIReq.ToolsChoice = input.ToolChoice + openAIReq.ToolsChoice = openAIToolChoice(input.ToolChoice) openAIReq.Metadata = input.Metadata _, tokenUsage, chatDeltas, err := openaiEndpoint.ComputeChoices(openAIReq, predInput, cfg, cl, appConfig, ml, func(s string, c *[]schema.Choice) {}, tokenCallback) @@ -1023,33 +1023,52 @@ func convertAnthropicTools(input *schema.AnthropicRequest, cfg *config.ModelConf funcs = append(funcs, f) } - // Handle tool_choice - if input.ToolChoice != nil { - switch tc := input.ToolChoice.(type) { - case string: - // "auto", "any", or "none" - if tc == "any" { - // Force the model to use one of the tools - cfg.SetFunctionCallString("required") - } else if tc == "none" { - // Don't use tools - return nil, false - } - // "auto" is the default - let model decide - case map[string]any: - // Specific tool selection: {"type": "tool", "name": "tool_name"} - if tcType, ok := tc["type"].(string); ok && tcType == "tool" { - if name, ok := tc["name"].(string); ok { - // Force specific tool - cfg.SetFunctionCallNameString(name) - } - } + switch mode, name := anthropicToolChoice(input.ToolChoice); mode { + case "any": + cfg.SetFunctionCallString("required") + case "none": + return nil, false + case "tool": + if name != "" { + cfg.SetFunctionCallNameString(name) } } return funcs, len(funcs) > 0 && cfg.ShouldUseFunctions() } +// anthropicToolChoice reads an Anthropic tool_choice, an object such as +// {"type": "any"} or {"type": "tool", "name": "get_weather"}. The bare +// strings "auto", "any" and "none" are accepted too. +func anthropicToolChoice(toolChoice any) (mode, name string) { + switch tc := toolChoice.(type) { + case string: + mode = tc + case map[string]any: + mode, _ = tc["type"].(string) + name, _ = tc["name"].(string) + } + return mode, name +} + +// openAIToolChoice translates an Anthropic tool_choice into the OpenAI form +// the backends read. Passing the Anthropic object through is not neutral: +// the llama.cpp backend turns any object tool_choice into "required", so +// {"type": "auto"} and {"type": "none"} would force a tool call. +func openAIToolChoice(toolChoice any) any { + switch mode, name := anthropicToolChoice(toolChoice); mode { + case "auto", "none": + return mode + case "any": + return "required" + case "tool": + if name != "" { + return map[string]any{"type": "function", "function": map[string]any{"name": name}} + } + } + return nil +} + // forwardCloudProxyAnthropicViaBackend marshals the Anthropic request, // and hands the body off to the cloud-proxy gRPC backend. Model swap + // upstream auth headers are applied inside the backend. Request-side PII diff --git a/core/http/endpoints/anthropic/messages_test.go b/core/http/endpoints/anthropic/messages_test.go index 160e7ecb07..ae5c77db45 100644 --- a/core/http/endpoints/anthropic/messages_test.go +++ b/core/http/endpoints/anthropic/messages_test.go @@ -1,9 +1,11 @@ package anthropic import ( + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" ) var _ = Describe("Anthropic thinking inbound", func() { @@ -122,3 +124,38 @@ var _ = Describe("Typed inbound conversion", func() { Expect(req.Messages[0].Content).To(Equal(blocks)) }) }) + +var _ = Describe("Anthropic tool_choice", func() { + tools := []schema.AnthropicTool{{Name: "get_weather"}} + + DescribeTable("convertAnthropicTools", + func(toolChoice any, expectTools bool, expectCall string) { + cfg := &config.ModelConfig{} + funcs, useFn := convertAnthropicTools(&schema.AnthropicRequest{Tools: tools, ToolChoice: toolChoice}, cfg) + Expect(useFn).To(Equal(expectTools)) + Expect(funcs != nil).To(Equal(expectTools)) + Expect(cfg.FunctionToCall()).To(Equal(expectCall)) + }, + Entry("absent", nil, true, ""), + Entry("auto object", map[string]any{"type": "auto"}, true, ""), + Entry("any object", map[string]any{"type": "any"}, true, "required"), + Entry("none object", map[string]any{"type": "none"}, false, ""), + Entry("tool object", map[string]any{"type": "tool", "name": "get_weather"}, true, "get_weather"), + Entry("any string", "any", true, "required"), + Entry("none string", "none", false, ""), + ) + + DescribeTable("openAIToolChoice", + func(toolChoice any, expected types.GomegaMatcher) { + Expect(openAIToolChoice(toolChoice)).To(expected) + }, + Entry("absent", nil, BeNil()), + Entry("auto", map[string]any{"type": "auto"}, Equal("auto")), + Entry("any", map[string]any{"type": "any", "disable_parallel_tool_use": true}, Equal("required")), + Entry("none", map[string]any{"type": "none"}, Equal("none")), + Entry("tool", map[string]any{"type": "tool", "name": "get_weather"}, + Equal(map[string]any{"type": "function", "function": map[string]any{"name": "get_weather"}})), + Entry("tool without a name", map[string]any{"type": "tool"}, BeNil()), + Entry("any string", "any", Equal("required")), + ) +}) diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index f7cf77be98..c062703c15 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -115,7 +115,7 @@ curl http://localhost:8080/v1/messages \ | `stop_sequences` | array | No | Array of strings that will stop generation | | `stream` | boolean | No | Enable streaming responses | | `tools` | array | No | Array of tool definitions for function calling | -| `tool_choice` | string/object | No | Tool choice strategy: "auto", "any", "none", or specific tool | +| `tool_choice` | object | No | Tool choice strategy: `{"type": "auto"}`, `{"type": "any"}`, `{"type": "none"}`, or `{"type": "tool", "name": "..."}`. The bare strings "auto", "any" and "none" are accepted too | | `metadata` | object | No | Per-request metadata passed to the backend (e.g., `{"enable_thinking": "true"}`) | #### Message Format @@ -176,7 +176,7 @@ curl http://localhost:8080/v1/messages \ } } ], - "tool_choice": "auto", + "tool_choice": {"type": "auto"}, "messages": [ {"role": "user", "content": "What is the weather in San Francisco?"} ] diff --git a/tests/e2e/e2e_anthropic_test.go b/tests/e2e/e2e_anthropic_test.go index e13801c02e..28389837d5 100644 --- a/tests/e2e/e2e_anthropic_test.go +++ b/tests/e2e/e2e_anthropic_test.go @@ -384,6 +384,59 @@ var _ = Describe("Anthropic API E2E test", func() { }) }) + Context("tool_choice", func() { + weatherTool := []anthropic.ToolUnionParam{{ + OfTool: &anthropic.ToolParam{ + Name: "get_weather", + InputSchema: anthropic.ToolInputSchemaParam{ + Type: constant.ValueOf[constant.Object](), + Properties: map[string]any{"location": map[string]any{"type": "string"}}, + }, + }, + }} + + It("does not call a tool when tool_choice is none", func() { + message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{ + Model: "mock-model", + MaxTokens: 1024, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("What's the weather like in San Francisco?")), + }, + Tools: weatherTool, + ToolChoice: anthropic.ToolChoiceUnionParam{OfNone: &anthropic.ToolChoiceNoneParam{}}, + }) + Expect(err).ToNot(HaveOccurred()) + for _, block := range message.Content { + Expect(string(block.Type)).ToNot(Equal("tool_use")) + } + Expect(string(message.StopReason)).To(Equal("end_turn")) + }) + + // The llama.cpp backend turns any object tool_choice into + // "required", so the Anthropic object must reach the backend + // in its OpenAI form. + DescribeTable("forwards tool_choice to the backend in OpenAI form", + func(toolChoice anthropic.ToolChoiceUnionParam, expected string) { + message, err := client.Messages.New(context.TODO(), anthropic.MessageNewParams{ + Model: "mock-model", + MaxTokens: 1024, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("ECHO_TOOL_CHOICE")), + }, + Tools: weatherTool, + ToolChoice: toolChoice, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(message.Content).ToNot(BeEmpty()) + Expect(message.Content[0].Text).To(Equal("TOOL_CHOICE=" + expected)) + }, + Entry("auto", anthropic.ToolChoiceUnionParam{OfAuto: &anthropic.ToolChoiceAutoParam{}}, `"auto"`), + Entry("any", anthropic.ToolChoiceUnionParam{OfAny: &anthropic.ToolChoiceAnyParam{}}, `"required"`), + Entry("none", anthropic.ToolChoiceUnionParam{OfNone: &anthropic.ToolChoiceNoneParam{}}, `"none"`), + Entry("tool", anthropic.ToolChoiceParamOfTool("get_weather"), `{"function":{"name":"get_weather"},"type":"function"}`), + ) + }) + Context("ChatDeltas (C++ autoparser)", func() { It("streams tool calls via ChatDeltas", func() { stream := client.Messages.NewStreaming(context.TODO(), anthropic.MessageNewParams{ diff --git a/tests/e2e/mock-backend/main.go b/tests/e2e/mock-backend/main.go index 05e31c9d77..f9ec8b7f74 100644 --- a/tests/e2e/mock-backend/main.go +++ b/tests/e2e/mock-backend/main.go @@ -219,6 +219,16 @@ func (m *MockBackend) Predict(ctx context.Context, in *pb.PredictOptions) (*pb.R }, nil } + // ECHO_TOOL_CHOICE returns the tool_choice the REST layer forwarded to + // the backend, so tests can pin the API -> gRPC translation. + if strings.Contains(in.Prompt, "ECHO_TOOL_CHOICE") { + return &pb.Reply{ + Message: []byte("TOOL_CHOICE=" + in.ToolChoice), + Tokens: 1, + PromptTokens: 1, + }, nil + } + // ECHO_SERVED_MODEL returns the loaded model file path so router e2e // tests can verify which candidate actually served the request without // adding a new RPC. The router fans out to a single backend process per