mirror of
https://github.com/mudler/LocalAI.git
synced 2026-05-30 11:36:31 -04:00
6a80e23733
Add a routing middleware stack and a cloud-proxy backend. * cloud-proxy: a Go gRPC backend that forwards OpenAI- and Anthropic-shaped chat requests to upstream providers, with an optional translate mode (OpenAI request -> Anthropic /v1/messages -> OpenAI response) and full tool-calling support. * routing: admission control, content-aware model routing (embedding cache + classifier + rerank + Arch-Router score), PII detection/redaction (regex + NER) with streaming filter and OpenAI/Anthropic adapters, and a per-user/per-key billing recorder backed by GORM or in-memory storage. * middleware: UsageMiddleware records usage via the billing recorder, plus admission, route-model, usage-stamp and trace middlewares. * observability: BackendTrace ring buffer stores full request bodies (capped), MITM proxy emits structured trace events, and router classifier decisions surface at /api/router/decide. * gallery: Arch-Router-1.5B (Q4_K_M and Q8_0). * UI: cloud-proxy model-editor fields, classifier system-prompt and score-normalization config, and a Traces page rendering request bodies. Assisted-by: claude-code:claude-opus-4-7 [Read] [Edit] [Bash] Signed-off-by: Richard Palethorpe <io@richiejp.com>
107 lines
3.3 KiB
Go
107 lines
3.3 KiB
Go
package openai
|
|
|
|
import (
|
|
"encoding/json"
|
|
"time"
|
|
|
|
"github.com/labstack/echo/v4"
|
|
"github.com/mudler/LocalAI/core/backend"
|
|
"github.com/mudler/LocalAI/core/config"
|
|
"github.com/mudler/LocalAI/core/http/middleware"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/mudler/LocalAI/core/schema"
|
|
|
|
"github.com/mudler/LocalAI/core/templates"
|
|
"github.com/mudler/LocalAI/pkg/model"
|
|
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
// EditEndpoint is the OpenAI edit API endpoint
|
|
// @Summary OpenAI edit endpoint
|
|
// @Tags inference
|
|
// @Param request body schema.OpenAIRequest true "query params"
|
|
// @Success 200 {object} schema.OpenAIResponse "Response"
|
|
// @Router /v1/edits [post]
|
|
func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig) echo.HandlerFunc {
|
|
|
|
return func(c echo.Context) error {
|
|
|
|
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
|
|
if !ok || input.Model == "" {
|
|
return echo.ErrBadRequest
|
|
}
|
|
// Opt-in extra usage flag
|
|
extraUsage := c.Request().Header.Get("Extra-Usage") != ""
|
|
|
|
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
|
if !ok || config == nil {
|
|
return echo.ErrBadRequest
|
|
}
|
|
|
|
xlog.Debug("Edit Endpoint Input", "input", input)
|
|
xlog.Debug("Edit Endpoint Config", "config", *config)
|
|
|
|
var result []schema.Choice
|
|
totalTokenUsage := backend.TokenUsage{}
|
|
|
|
for _, i := range config.InputStrings {
|
|
templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.EditPromptTemplate, *config, templates.PromptTemplateData{
|
|
Input: i,
|
|
Instruction: input.Instruction,
|
|
SystemPrompt: config.SystemPrompt,
|
|
ReasoningEffort: input.ReasoningEffort,
|
|
Metadata: input.Metadata,
|
|
})
|
|
if err == nil {
|
|
i = templatedInput
|
|
xlog.Debug("Template found, input modified", "input", i)
|
|
}
|
|
|
|
r, tokenUsage, _, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
|
|
*c = append(*c, schema.Choice{Text: s})
|
|
}, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
totalTokenUsage.Prompt += tokenUsage.Prompt
|
|
totalTokenUsage.Completion += tokenUsage.Completion
|
|
|
|
totalTokenUsage.TimingTokenGeneration += tokenUsage.TimingTokenGeneration
|
|
totalTokenUsage.TimingPromptProcessing += tokenUsage.TimingPromptProcessing
|
|
|
|
result = append(result, r...)
|
|
}
|
|
usage := schema.OpenAIUsage{
|
|
PromptTokens: totalTokenUsage.Prompt,
|
|
CompletionTokens: totalTokenUsage.Completion,
|
|
TotalTokens: totalTokenUsage.Prompt + totalTokenUsage.Completion,
|
|
}
|
|
if extraUsage {
|
|
usage.TimingTokenGeneration = totalTokenUsage.TimingTokenGeneration
|
|
usage.TimingPromptProcessing = totalTokenUsage.TimingPromptProcessing
|
|
}
|
|
|
|
id := uuid.New().String()
|
|
created := int(time.Now().Unix())
|
|
resp := &schema.OpenAIResponse{
|
|
ID: id,
|
|
Created: created,
|
|
Model: input.Model, // we have to return what the user sent here, due to OpenAI spec.
|
|
Choices: result,
|
|
Object: "edit",
|
|
Usage: &usage,
|
|
}
|
|
|
|
jsonResult, _ := json.Marshal(resp)
|
|
xlog.Debug("Response", "response", string(jsonResult))
|
|
|
|
middleware.StampUsage(c, input.Model, totalTokenUsage.Prompt, totalTokenUsage.Completion)
|
|
|
|
// Return the prediction in the response body
|
|
return c.JSON(200, resp)
|
|
}
|
|
}
|