Files
LocalAI/pkg/mcp/localaitools/tools_middleware.go
Richard Palethorpe 6a80e23733 feat(middleware): Model routing, PII filtering, Cloud model proxies (#9802)
Add a routing middleware stack and a cloud-proxy backend.

* cloud-proxy: a Go gRPC backend that forwards OpenAI- and
  Anthropic-shaped chat requests to upstream providers, with an
  optional translate mode (OpenAI request -> Anthropic /v1/messages
  -> OpenAI response) and full tool-calling support.

* routing: admission control, content-aware model routing
  (embedding cache + classifier + rerank + Arch-Router score),
  PII detection/redaction (regex + NER) with streaming filter and
  OpenAI/Anthropic adapters, and a per-user/per-key billing recorder
  backed by GORM or in-memory storage.

* middleware: UsageMiddleware records usage via the billing recorder,
  plus admission, route-model, usage-stamp and trace middlewares.

* observability: BackendTrace ring buffer stores full request bodies
  (capped), MITM proxy emits structured trace events, and router
  classifier decisions surface at /api/router/decide.

* gallery: Arch-Router-1.5B (Q4_K_M and Q8_0).

* UI: cloud-proxy model-editor fields, classifier system-prompt and
  score-normalization config, and a Traces page rendering request
  bodies.

Assisted-by: claude-code:claude-opus-4-7 [Read] [Edit] [Bash]

Signed-off-by: Richard Palethorpe <io@richiejp.com>
2026-05-25 09:28:27 +02:00

79 lines
3.5 KiB
Go

package localaitools
import (
"context"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// registerMiddlewareTools wires the routing-module admin surface for the
// MCP server. The two tools mirror what the React /app/middleware page
// exposes:
//
// - get_middleware_status: read-only aggregator. The agent can ask
// "what's filtering my requests?" and get back the active PII
// pattern set, the per-model resolved enabled/override state, and
// a placeholder for routing.
// - set_pii_pattern_action: mutating. Mutations are TRANSIENT — they
// live until process restart, when patterns reload from the YAML
// defaults. The skill prompt should warn the user about that
// before applying lasting changes.
func registerMiddlewareTools(s *mcp.Server, client LocalAIClient, opts Options) {
mcp.AddTool(s, &mcp.Tool{
Name: ToolGetMiddlewareStatus,
Description: "Aggregated routing-module status: PII pattern catalogue with current actions, per-model resolved PII state and overrides, recent event count, plus the active router models and their classifier configs. Read-only.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
status, err := client.GetMiddlewareStatus(ctx)
if err != nil {
return errorResult(err), nil, nil
}
return jsonResult(status), nil, nil
})
mcp.AddTool(s, &mcp.Tool{
Name: ToolGetRouterDecisions,
Description: "Recent intelligent-routing decisions. Each row records which router model the client called, which candidate the classifier picked, the classifier's score and latency, and a correlation id that joins back to the usage record. Filter by correlation_id, user_id, or router_model. Read-only.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, args RouterDecisionsQuery) (*mcp.CallToolResult, any, error) {
decisions, err := client.GetRouterDecisions(ctx, args)
if err != nil {
return errorResult(err), nil, nil
}
return jsonResult(decisions), nil, nil
})
if opts.DisableMutating {
return
}
mcp.AddTool(s, &mcp.Tool{
Name: ToolSetPIIPatternAction,
Description: "Change a PII pattern's action (mask|block|route_local) and/or disabled state in-process. TRANSIENT: the mutation is lost on restart unless followed by persist_pii_patterns. Admin-required.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, args PIIPatternActionUpdate) (*mcp.CallToolResult, any, error) {
if args.ID == "" {
return errorResultf("id is required"), nil, nil
}
if args.Action == "" && args.Disabled == nil {
return errorResultf("at least one of action (mask, block, route_local) or disabled must be set"), nil, nil
}
if err := client.SetPIIPatternAction(ctx, args); err != nil {
return errorResult(err), nil, nil
}
return jsonResult(map[string]any{
"id": args.ID,
"action": args.Action,
"disabled": args.Disabled,
"persisted": false,
}), nil, nil
})
mcp.AddTool(s, &mcp.Tool{
Name: ToolPersistPIIPatterns,
Description: "Snapshot the live PII redactor's per-pattern (action, disabled) state into runtime_settings.json so it re-applies on the next process start. Pairs with set_pii_pattern_action — that one is in-process; this one persists. Admin-required.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
if err := client.PersistPIIPatterns(ctx); err != nil {
return errorResult(err), nil, nil
}
return jsonResult(map[string]any{"persisted": true}), nil, nil
})
}