mirror of
https://github.com/mudler/LocalAI.git
synced 2026-06-11 18:27:32 -04:00
Add a routing middleware stack and a cloud-proxy backend. * cloud-proxy: a Go gRPC backend that forwards OpenAI- and Anthropic-shaped chat requests to upstream providers, with an optional translate mode (OpenAI request -> Anthropic /v1/messages -> OpenAI response) and full tool-calling support. * routing: admission control, content-aware model routing (embedding cache + classifier + rerank + Arch-Router score), PII detection/redaction (regex + NER) with streaming filter and OpenAI/Anthropic adapters, and a per-user/per-key billing recorder backed by GORM or in-memory storage. * middleware: UsageMiddleware records usage via the billing recorder, plus admission, route-model, usage-stamp and trace middlewares. * observability: BackendTrace ring buffer stores full request bodies (capped), MITM proxy emits structured trace events, and router classifier decisions surface at /api/router/decide. * gallery: Arch-Router-1.5B (Q4_K_M and Q8_0). * UI: cloud-proxy model-editor fields, classifier system-prompt and score-normalization config, and a Traces page rendering request bodies. Assisted-by: claude-code:claude-opus-4-7 [Read] [Edit] [Bash] Signed-off-by: Richard Palethorpe <io@richiejp.com>
79 lines
3.5 KiB
Go
79 lines
3.5 KiB
Go
package localaitools
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/modelcontextprotocol/go-sdk/mcp"
|
|
)
|
|
|
|
// registerMiddlewareTools wires the routing-module admin surface for the
|
|
// MCP server. The two tools mirror what the React /app/middleware page
|
|
// exposes:
|
|
//
|
|
// - get_middleware_status: read-only aggregator. The agent can ask
|
|
// "what's filtering my requests?" and get back the active PII
|
|
// pattern set, the per-model resolved enabled/override state, and
|
|
// a placeholder for routing.
|
|
// - set_pii_pattern_action: mutating. Mutations are TRANSIENT — they
|
|
// live until process restart, when patterns reload from the YAML
|
|
// defaults. The skill prompt should warn the user about that
|
|
// before applying lasting changes.
|
|
func registerMiddlewareTools(s *mcp.Server, client LocalAIClient, opts Options) {
|
|
mcp.AddTool(s, &mcp.Tool{
|
|
Name: ToolGetMiddlewareStatus,
|
|
Description: "Aggregated routing-module status: PII pattern catalogue with current actions, per-model resolved PII state and overrides, recent event count, plus the active router models and their classifier configs. Read-only.",
|
|
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
|
|
status, err := client.GetMiddlewareStatus(ctx)
|
|
if err != nil {
|
|
return errorResult(err), nil, nil
|
|
}
|
|
return jsonResult(status), nil, nil
|
|
})
|
|
|
|
mcp.AddTool(s, &mcp.Tool{
|
|
Name: ToolGetRouterDecisions,
|
|
Description: "Recent intelligent-routing decisions. Each row records which router model the client called, which candidate the classifier picked, the classifier's score and latency, and a correlation id that joins back to the usage record. Filter by correlation_id, user_id, or router_model. Read-only.",
|
|
}, func(ctx context.Context, _ *mcp.CallToolRequest, args RouterDecisionsQuery) (*mcp.CallToolResult, any, error) {
|
|
decisions, err := client.GetRouterDecisions(ctx, args)
|
|
if err != nil {
|
|
return errorResult(err), nil, nil
|
|
}
|
|
return jsonResult(decisions), nil, nil
|
|
})
|
|
|
|
if opts.DisableMutating {
|
|
return
|
|
}
|
|
|
|
mcp.AddTool(s, &mcp.Tool{
|
|
Name: ToolSetPIIPatternAction,
|
|
Description: "Change a PII pattern's action (mask|block|route_local) and/or disabled state in-process. TRANSIENT: the mutation is lost on restart unless followed by persist_pii_patterns. Admin-required.",
|
|
}, func(ctx context.Context, _ *mcp.CallToolRequest, args PIIPatternActionUpdate) (*mcp.CallToolResult, any, error) {
|
|
if args.ID == "" {
|
|
return errorResultf("id is required"), nil, nil
|
|
}
|
|
if args.Action == "" && args.Disabled == nil {
|
|
return errorResultf("at least one of action (mask, block, route_local) or disabled must be set"), nil, nil
|
|
}
|
|
if err := client.SetPIIPatternAction(ctx, args); err != nil {
|
|
return errorResult(err), nil, nil
|
|
}
|
|
return jsonResult(map[string]any{
|
|
"id": args.ID,
|
|
"action": args.Action,
|
|
"disabled": args.Disabled,
|
|
"persisted": false,
|
|
}), nil, nil
|
|
})
|
|
|
|
mcp.AddTool(s, &mcp.Tool{
|
|
Name: ToolPersistPIIPatterns,
|
|
Description: "Snapshot the live PII redactor's per-pattern (action, disabled) state into runtime_settings.json so it re-applies on the next process start. Pairs with set_pii_pattern_action — that one is in-process; this one persists. Admin-required.",
|
|
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
|
|
if err := client.PersistPIIPatterns(ctx); err != nil {
|
|
return errorResult(err), nil, nil
|
|
}
|
|
return jsonResult(map[string]any{"persisted": true}), nil, nil
|
|
})
|
|
}
|