mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-30 10:09:13 -04:00
Ettore Di Giacinto
10f7b1ca60
The proxy sent every chat and completion request with context.Background, and the gRPC server gave rich backends no context at all. When a client disconnected, or failover gave up on the target, the upstream kept generating to the end, which costs tokens on a paid or shared upstream. A silent upstream held the backend forever. Add the optional AIModelRichContext interface. The gRPC server prefers it and passes the call's context, like the Score and Rerank extensions. The proxy implements it, so the upstream request ends with the gRPC call. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Assisted-by: Claude:claude-opus-5-5 [Claude Code]
37 lines
1.2 KiB
Go
37 lines
1.2 KiB
Go
package main
|
|
|
|
// localai-proxy is a LocalAI backend that serves backend gRPC methods by
|
|
// calling the REST API of another LocalAI instance. It lets a model config
|
|
// (and so a failover chain target) live on a remote LocalAI while callers
|
|
// keep using the local backend interface for every modality, not only chat.
|
|
|
|
import (
|
|
"flag"
|
|
"os"
|
|
|
|
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
|
"github.com/mudler/xlog"
|
|
"golang.org/x/term"
|
|
)
|
|
|
|
var addr = flag.String("addr", "localhost:50051", "the address to listen on")
|
|
|
|
func main() {
|
|
// xlog's default handler emits ANSI color codes, which are unreadable once
|
|
// LocalAI captures the backend's stdout into a log file. Force plain text
|
|
// when LOCALAI_LOG_FORMAT is unset and stdout is not a terminal.
|
|
format := os.Getenv("LOCALAI_LOG_FORMAT")
|
|
if format == "" && !term.IsTerminal(int(os.Stdout.Fd())) {
|
|
format = xlog.TextFormat
|
|
}
|
|
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(os.Getenv("LOCALAI_LOG_LEVEL")), format))
|
|
flag.Parse()
|
|
if err := grpc.StartServer(*addr, NewLocalAIProxy()); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// The gRPC server only passes the call's context to backends that implement
|
|
// this, so keep the proxy on it.
|
|
var _ grpc.AIModelRichContext = (*LocalAIProxy)(nil)
|