mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-08 23:37:43 -04:00
6a80e23733
Add a routing middleware stack and a cloud-proxy backend. * cloud-proxy: a Go gRPC backend that forwards OpenAI- and Anthropic-shaped chat requests to upstream providers, with an optional translate mode (OpenAI request -> Anthropic /v1/messages -> OpenAI response) and full tool-calling support. * routing: admission control, content-aware model routing (embedding cache + classifier + rerank + Arch-Router score), PII detection/redaction (regex + NER) with streaming filter and OpenAI/Anthropic adapters, and a per-user/per-key billing recorder backed by GORM or in-memory storage. * middleware: UsageMiddleware records usage via the billing recorder, plus admission, route-model, usage-stamp and trace middlewares. * observability: BackendTrace ring buffer stores full request bodies (capped), MITM proxy emits structured trace events, and router classifier decisions surface at /api/router/decide. * gallery: Arch-Router-1.5B (Q4_K_M and Q8_0). * UI: cloud-proxy model-editor fields, classifier system-prompt and score-normalization config, and a Traces page rendering request bodies. Assisted-by: claude-code:claude-opus-4-7 [Read] [Edit] [Bash] Signed-off-by: Richard Palethorpe <io@richiejp.com>
77 lines
2.2 KiB
Go
77 lines
2.2 KiB
Go
package router
|
|
|
|
import (
|
|
"sync"
|
|
)
|
|
|
|
// Registry is the process-wide store of built classifiers, keyed by
|
|
// router-model name. The middleware uses it to avoid rebuilding the
|
|
// score classifier on every request, and the admin status endpoint
|
|
// reads from it to surface per-classifier cache stats.
|
|
//
|
|
// Each entry carries the fingerprint of the RouterConfig it was built
|
|
// from. A Get() with a stale fingerprint reports a miss so the
|
|
// middleware rebuilds — matches the previous local-sync.Map behaviour
|
|
// that keyed on fingerprint alone.
|
|
type Registry struct {
|
|
entries sync.Map // name → *registryEntry
|
|
}
|
|
|
|
type registryEntry struct {
|
|
fingerprint uint64
|
|
classifier Classifier
|
|
}
|
|
|
|
func NewRegistry() *Registry { return &Registry{} }
|
|
|
|
// Get returns the cached classifier for the named router model iff the
|
|
// stored fingerprint matches. A miss (no entry, or stale fingerprint)
|
|
// returns false; the caller is expected to rebuild and Put the result.
|
|
func (r *Registry) Get(name string, fingerprint uint64) (Classifier, bool) {
|
|
if r == nil {
|
|
return nil, false
|
|
}
|
|
v, ok := r.entries.Load(name)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
e := v.(*registryEntry)
|
|
if e.fingerprint != fingerprint {
|
|
return nil, false
|
|
}
|
|
return e.classifier, true
|
|
}
|
|
|
|
// Put stores a built classifier under (name, fingerprint), replacing
|
|
// any prior entry. The middleware calls this after a Get miss.
|
|
func (r *Registry) Put(name string, fingerprint uint64, c Classifier) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
r.entries.Store(name, ®istryEntry{fingerprint: fingerprint, classifier: c})
|
|
}
|
|
|
|
// EmbeddingCacheStatsByRouter returns a snapshot of every embedding
|
|
// cache currently in the registry, keyed by router-model name. Plain
|
|
// classifiers without the L2 cache wrapper are skipped — callers
|
|
// distinguish "cache disabled" from "cache enabled with zero hits" by
|
|
// the presence of the map key.
|
|
func (r *Registry) EmbeddingCacheStatsByRouter() map[string]EmbeddingCacheStats {
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
out := map[string]EmbeddingCacheStats{}
|
|
r.entries.Range(func(k, v any) bool {
|
|
name, _ := k.(string)
|
|
e, _ := v.(*registryEntry)
|
|
if e == nil {
|
|
return true
|
|
}
|
|
if ec, ok := e.classifier.(*EmbeddingCacheClassifier); ok {
|
|
out[name] = ec.Stats()
|
|
}
|
|
return true
|
|
})
|
|
return out
|
|
}
|