From 4b4faa4ac7c744830fdd9d8f8609c220212437a0 Mon Sep 17 00:00:00 2001 From: walcz-de Date: Tue, 28 Jul 2026 19:38:14 +0200 Subject: [PATCH] feat(cloud-proxy): optional Anthropic prompt-cache breakpoints in translate mode (#11158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Anthropic translate provider builds the upstream request from scratch and never emitted cache_control, so prompt caching was impossible for OpenAI-format clients routed through cloud-proxy — even though the entire system prompt + tools prefix is re-sent on every agentic turn. Add an opt-in cache_prompt flag (ProxyOptions.cache_prompt; model YAML proxy.cache_prompt: true). On a translate+anthropic model, buildAnthropicRequest injects cache_control:{type:ephemeral} on the stable prefix — the system block, the last tool, and the last message block (at most 3 of Anthropic's 4 allowed breakpoints). Anthropic then serves the repeated prefix at the cache-read rate (0.1x input) on subsequent calls, cutting cost on multi-turn/agentic workloads. No effect in passthrough mode, for non-Anthropic providers, or when unset. System is widened to any so it can carry the block form required to attach cache_control, while still marshalling as a bare string when caching is off. Adds a unit test asserting exactly three breakpoints when on and none when off, and documents the option in docs/content/operations/cloud-proxy.md. Assisted-by: Claude:opus-4.8 Signed-off-by: stefanwalcz --- backend/backend.proto | 6 ++ backend/go/cloud-proxy/provider_anthropic.go | 82 +++++++++++++++++-- .../go/cloud-proxy/provider_anthropic_test.go | 59 +++++++++++++ backend/go/cloud-proxy/proxy.go | 2 + core/backend/options.go | 1 + core/config/meta/registry.go | 7 ++ core/config/model_config.go | 9 ++ docs/content/operations/cloud-proxy.md | 28 +++++++ 8 files changed, 187 insertions(+), 7 deletions(-) diff --git a/backend/backend.proto b/backend/backend.proto index 25f45eac8..334b526d7 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -508,6 +508,12 @@ message ProxyOptions { string api_key_file = 5; string upstream_model = 6; int32 request_timeout_seconds = 7; + // cache_prompt enables automatic Anthropic prompt-cache breakpoints + // (cache_control: ephemeral) on the stable prefix — system, tools, and + // the last message block — when translating to the Anthropic provider. + // Cuts input cost on repeated/agentic calls (cache read = 0.1x). Only + // meaningful for mode=translate + provider=anthropic; ignored otherwise. + bool cache_prompt = 8; } message Result { diff --git a/backend/go/cloud-proxy/provider_anthropic.go b/backend/go/cloud-proxy/provider_anthropic.go index d86f2ab8e..aa39e1ccb 100644 --- a/backend/go/cloud-proxy/provider_anthropic.go +++ b/backend/go/cloud-proxy/provider_anthropic.go @@ -32,7 +32,9 @@ import ( type anthropicRequest struct { Model string `json:"model"` MaxTokens int32 `json:"max_tokens"` - System string `json:"system,omitempty"` + // System is `any`: a bare string normally, or []anthropicSystemBlock + // when cache_prompt is on (the block form carries cache_control). + System any `json:"system,omitempty"` Messages []anthropicMessage `json:"messages"` Stream bool `json:"stream,omitempty"` Temperature *float64 `json:"temperature,omitempty"` @@ -52,9 +54,30 @@ type anthropicMessage struct { } type anthropicTool struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - InputSchema json.RawMessage `json:"input_schema"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema json.RawMessage `json:"input_schema"` + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` +} + +// anthropicCacheControl marks a prompt-cache breakpoint. Anthropic caches +// everything up to and including a block tagged {"type":"ephemeral"} (5-min +// TTL) and serves that prefix at the cache-read rate (0.1x input) on later +// calls that share it — the win on agentic/multi-turn workloads. +type anthropicCacheControl struct { + Type string `json:"type"` // "ephemeral" +} + +// ephemeralCacheControl is the single reused breakpoint marker. +var ephemeralCacheControl = &anthropicCacheControl{Type: "ephemeral"} + +// anthropicSystemBlock is the block form of the top-level system field. +// Anthropic accepts system as a bare string OR a list of text blocks; the +// block form is required to attach cache_control to the system prompt. +type anthropicSystemBlock struct { + Type string `json:"type"` // "text" + Text string `json:"text"` + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` } // anthropicToolChoice mirrors the four shapes Anthropic accepts: @@ -81,8 +104,9 @@ type anthropicContentBlock struct { // Tool-result block fields. tool_result uses `content` (not // `text`) and pairs with `tool_use_id`; modelling them as // distinct fields avoids ambiguity at marshal time. - ToolUseID string `json:"tool_use_id,omitempty"` - ResultContent string `json:"content,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + ResultContent string `json:"content,omitempty"` + CacheControl *anthropicCacheControl `json:"cache_control,omitempty"` } type anthropicResponse struct { @@ -156,6 +180,11 @@ func buildAnthropicRequest(opts *pb.PredictOptions, cfg *proxyConfig, stream boo if req.ToolChoice != nil && req.ToolChoice.Type == anthropicToolChoiceNone { req.Tools, req.ToolChoice = nil, nil } + // Prompt-cache breakpoint on the last tool: Anthropic caches the entire + // tool block up to the marked tool — usually a large, fully stable prefix. + if cfg.cachePrompt && len(req.Tools) > 0 { + req.Tools[len(req.Tools)-1].CacheControl = ephemeralCacheControl + } var systemParts []string for _, m := range opts.GetMessages() { @@ -189,15 +218,54 @@ func buildAnthropicRequest(opts *pb.PredictOptions, cfg *proxyConfig, stream boo }) } } - req.System = strings.Join(systemParts, "\n\n") + // System: block form (with cache_control) when caching is on, else the + // bare string. Only set when non-empty so `omitempty` still drops it. + if len(systemParts) > 0 { + joined := strings.Join(systemParts, "\n\n") + if cfg.cachePrompt { + req.System = []anthropicSystemBlock{{Type: "text", Text: joined, CacheControl: ephemeralCacheControl}} + } else { + req.System = joined + } + } if len(req.Messages) == 0 && opts.GetPrompt() != "" { req.Messages = []anthropicMessage{{Role: "user", Content: opts.GetPrompt()}} } + // Prompt-cache breakpoint on the final message block caches the whole + // conversation prefix up to the newest turn. With the system + tools + // breakpoints above, Anthropic serves the entire stable head at the + // cache-read rate on the next agentic iteration (max 4 breakpoints; we + // use at most 3, so we never exceed the limit). + if cfg.cachePrompt { + markLastMessageCacheable(req.Messages) + } + return json.Marshal(req) } +// markLastMessageCacheable tags the final block of the last message with a +// cache_control breakpoint. String content is promoted to a single text +// block so the marker has somewhere to attach; block content gets the marker +// on its last element. +func markLastMessageCacheable(msgs []anthropicMessage) { + if len(msgs) == 0 { + return + } + last := &msgs[len(msgs)-1] + switch c := last.Content.(type) { + case string: + if c != "" { + last.Content = []anthropicContentBlock{{Type: "text", Text: c, CacheControl: ephemeralCacheControl}} + } + case []anthropicContentBlock: + if len(c) > 0 { + c[len(c)-1].CacheControl = ephemeralCacheControl + } + } +} + // appendToolResult appends a tool_result block as a user message, // merging into a preceding user message that already carries blocks. // Anthropic concatenates consecutive same-role messages on its end, diff --git a/backend/go/cloud-proxy/provider_anthropic_test.go b/backend/go/cloud-proxy/provider_anthropic_test.go index 6119c97cf..2ed9a2127 100644 --- a/backend/go/cloud-proxy/provider_anthropic_test.go +++ b/backend/go/cloud-proxy/provider_anthropic_test.go @@ -328,3 +328,62 @@ func TestBuildAnthropic_RoundTripsAssistantToolCalls(t *testing.T) { g.Expect(r0["tool_use_id"]).To(Equal("call_abc")) g.Expect(r0["content"]).To(Equal(`{"models":["a","b"]}`)) } + +// TestPredict_Anthropic_PromptCache verifies that cache_prompt injects +// exactly the intended cache_control breakpoints (system, last tool, last +// message) when on, and none when off — asserting on the raw upstream body +// because System becomes a block list that the typed struct hides. +func TestPredict_Anthropic_PromptCache(t *testing.T) { + g := NewWithT(t) + + // run issues one translate Predict and returns the raw body the fake + // Anthropic upstream received. + run := func(cachePrompt bool) string { + var rawBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + rawBody = string(b) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"m","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-3-5-sonnet-20241022","usage":{"input_tokens":5,"output_tokens":2}}`) + })) + defer srv.Close() + + t.Setenv("CLOUD_PROXY_ANTHROPIC_FAKE", "sk-ant-fake") + cp := NewCloudProxy() + err := cp.Load(&pb.ModelOptions{ + Model: "claude-local", + Proxy: &pb.ProxyOptions{ + UpstreamUrl: srv.URL, + Mode: modeTranslate, + Provider: providerAnthropic, + ApiKeyEnv: "CLOUD_PROXY_ANTHROPIC_FAKE", + UpstreamModel: "claude-3-5-sonnet-20241022", + CachePrompt: cachePrompt, + }, + }) + g.Expect(err).NotTo(HaveOccurred()) + + _, err = cp.Predict(&pb.PredictOptions{ + Messages: []*pb.Message{ + {Role: "system", Content: "be brief"}, + {Role: "user", Content: "hello"}, + }, + Tools: `[{"type":"function","function":{"name":"t","parameters":{"type":"object"}}}]`, + Tokens: 32, + }) + g.Expect(err).NotTo(HaveOccurred()) + return rawBody + } + + // cache_prompt ON: three ephemeral breakpoints (system + last tool + + // last message), and system is emitted in block form. + on := run(true) + g.Expect(strings.Count(on, `"cache_control":{"type":"ephemeral"}`)).To(Equal(3), + "expected 3 breakpoints (system, tool, last message); body=%s", on) + g.Expect(on).To(ContainSubstring(`"system":[{"type":"text","text":"be brief"`)) + + // cache_prompt OFF: no breakpoints, system stays a bare string. + off := run(false) + g.Expect(off).NotTo(ContainSubstring("cache_control")) + g.Expect(off).To(ContainSubstring(`"system":"be brief"`)) +} diff --git a/backend/go/cloud-proxy/proxy.go b/backend/go/cloud-proxy/proxy.go index 4541ed772..f6e3a9f1b 100644 --- a/backend/go/cloud-proxy/proxy.go +++ b/backend/go/cloud-proxy/proxy.go @@ -48,6 +48,7 @@ type proxyConfig struct { upstreamModel string localModel string // ModelOptions.Model — fallback when upstream_model is unset apiKey string // resolved at Load time + cachePrompt bool // inject Anthropic prompt-cache breakpoints (translate+anthropic) } func NewCloudProxy() *CloudProxy { @@ -106,6 +107,7 @@ func (c *CloudProxy) Load(opts *pb.ModelOptions) error { upstreamModel: po.GetUpstreamModel(), localModel: opts.GetModel(), apiKey: key, + cachePrompt: po.GetCachePrompt(), }) xlog.Info("cloud-proxy: ready", "upstream", po.GetUpstreamUrl(), diff --git a/core/backend/options.go b/core/backend/options.go index 72f49f1ed..138a236aa 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -475,6 +475,7 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions { ApiKeyFile: c.Proxy.APIKeyFile, UpstreamModel: c.Proxy.UpstreamModel, RequestTimeoutSeconds: int32(c.Proxy.RequestTimeoutSeconds), + CachePrompt: c.Proxy.CachePrompt, } } diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 2ad64f4d3..56df838db 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -822,6 +822,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Min: f64(0), Order: 213, }, + "proxy.cache_prompt": { + Section: "proxy", + Label: "Proxy Anthropic Prompt Cache", + Description: "Inject Anthropic prompt-cache breakpoints (cache_control: ephemeral) on the stable prefix (system, tools, last message) when mode is translate and provider is anthropic. Serves the repeated prefix at the cache-read rate on multi-turn/agentic calls. No effect otherwise.", + Component: "checkbox", + Order: 214, + }, // --- MITM intercept hosts --- // Each host listed here is claimed by this model config; the diff --git a/core/config/model_config.go b/core/config/model_config.go index 7ca24be64..e82f84f0e 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -232,6 +232,15 @@ type ProxyConfig struct { // means no per-request timeout (only the request context, which // is bound to the client connection, applies). RequestTimeoutSeconds int `yaml:"request_timeout_seconds,omitempty" json:"request_timeout_seconds,omitempty"` + + // CachePrompt enables automatic Anthropic prompt-cache breakpoints + // (cache_control: ephemeral) on the stable prefix — system prompt, + // tools, and the last message block — when mode=translate and + // provider=anthropic. Anthropic then serves the repeated prefix at + // the cache-read rate (0.1x input), which sharply cuts cost on + // agentic/multi-turn workloads that re-send a large stable prefix. + // No effect for passthrough mode or non-Anthropic providers. + CachePrompt bool `yaml:"cache_prompt,omitempty" json:"cache_prompt,omitempty"` } // Proxy mode names. Validate() normalises an empty Mode to diff --git a/docs/content/operations/cloud-proxy.md b/docs/content/operations/cloud-proxy.md index df310eb7d..02af25bd0 100644 --- a/docs/content/operations/cloud-proxy.md +++ b/docs/content/operations/cloud-proxy.md @@ -164,6 +164,34 @@ image blocks, and per-request usage tokens are dropped through the internal `Predict()` signature. Use passthrough mode when your clients need the upstream's full feature set. +#### Anthropic prompt caching + +`proxy.cache_prompt: true` makes the translator add Anthropic +[prompt-cache](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) +breakpoints (`cache_control: {type: ephemeral}`) to the stable prefix of every +request: the system block, the last tool, and the final message block (at most +three of Anthropic's four allowed breakpoints). Anthropic then serves that +repeated prefix at the cache-read rate (~0.1x input) on subsequent calls, which +sharply cuts cost on agentic or multi-turn workloads that re-send a large, +unchanging system-plus-tools prefix each turn. + +The flag only applies with `mode: translate` and `provider: anthropic`; it has +no effect in passthrough mode, for other providers, or when unset (the system +field is then still emitted as a bare string). + +```yaml +name: claude-cached +backend: cloud-proxy + +proxy: + mode: translate + provider: anthropic + upstream_url: https://api.anthropic.com/v1/messages + api_key_env: ANTHROPIC_API_KEY + upstream_model: claude-3-5-sonnet-20241022 + cache_prompt: true +``` + ## Loading secrets from a file `api_key_file` is an alternative to `api_key_env` when your secret manager