diff --git a/core/backend/embeddings.go b/core/backend/embeddings.go
index 2383023c0..2e1f6c4b0 100644
--- a/core/backend/embeddings.go
+++ b/core/backend/embeddings.go
@@ -2,8 +2,10 @@ package backend
import (
"fmt"
+ "time"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/pkg/grpc"
model "github.com/mudler/LocalAI/pkg/model"
@@ -53,7 +55,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConf
}
}
- return func() ([]float32, error) {
+ wrappedFn := func() ([]float32, error) {
embeds, err := fn()
if err != nil {
return embeds, err
@@ -67,5 +69,48 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConf
}
}
return embeds, nil
- }, nil
+ }
+
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+
+ traceData := map[string]any{
+ "input_text": trace.TruncateString(s, 1000),
+ "input_tokens_count": len(tokens),
+ }
+
+ startTime := time.Now()
+ originalFn := wrappedFn
+ wrappedFn = func() ([]float32, error) {
+ result, err := originalFn()
+ duration := time.Since(startTime)
+
+ traceData["embedding_dimensions"] = len(result)
+
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ summary := trace.TruncateString(s, 200)
+ if summary == "" {
+ summary = fmt.Sprintf("tokens[%d]", len(tokens))
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: duration,
+ Type: trace.BackendTraceEmbedding,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: summary,
+ Error: errStr,
+ Data: traceData,
+ })
+
+ return result, err
+ }
+ }
+
+ return wrappedFn, nil
}
diff --git a/core/backend/image.go b/core/backend/image.go
index 651293cf5..89ea14cb1 100644
--- a/core/backend/image.go
+++ b/core/backend/image.go
@@ -1,7 +1,10 @@
package backend
import (
+ "time"
+
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/pkg/grpc/proto"
model "github.com/mudler/LocalAI/pkg/model"
@@ -36,6 +39,46 @@ func ImageGeneration(height, width, step, seed int, positive_prompt, negative_pr
return err
}
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+
+ traceData := map[string]any{
+ "positive_prompt": positive_prompt,
+ "negative_prompt": negative_prompt,
+ "height": height,
+ "width": width,
+ "step": step,
+ "seed": seed,
+ "source_image": src,
+ "destination": dst,
+ }
+
+ startTime := time.Now()
+ originalFn := fn
+ fn = func() error {
+ err := originalFn()
+ duration := time.Since(startTime)
+
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: duration,
+ Type: trace.BackendTraceImageGeneration,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(positive_prompt, 200),
+ Error: errStr,
+ Data: traceData,
+ })
+
+ return err
+ }
+ }
+
return fn, nil
}
diff --git a/core/backend/llm.go b/core/backend/llm.go
index 18367cb2f..40b53e74c 100644
--- a/core/backend/llm.go
+++ b/core/backend/llm.go
@@ -7,11 +7,13 @@ import (
"slices"
"strings"
"sync"
+ "time"
"unicode/utf8"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services"
@@ -220,6 +222,84 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
}
}
+ if o.EnableTracing {
+ trace.InitBackendTracingIfEnabled(o.TracingMaxItems)
+
+ traceData := map[string]any{
+ "prompt": s,
+ "use_tokenizer_template": c.TemplateConfig.UseTokenizerTemplate,
+ "chat_template": c.TemplateConfig.Chat,
+ "function_template": c.TemplateConfig.Functions,
+ "grammar": c.Grammar,
+ "stop_words": c.StopWords,
+ "streaming": tokenCallback != nil,
+ "images_count": len(images),
+ "videos_count": len(videos),
+ "audios_count": len(audios),
+ }
+
+ if len(messages) > 0 {
+ if msgJSON, err := json.Marshal(messages); err == nil {
+ traceData["messages"] = string(msgJSON)
+ }
+ }
+ if tools != "" {
+ traceData["tools"] = tools
+ }
+ if toolChoice != "" {
+ traceData["tool_choice"] = toolChoice
+ }
+ if reasoningJSON, err := json.Marshal(c.ReasoningConfig); err == nil {
+ traceData["reasoning_config"] = string(reasoningJSON)
+ }
+ traceData["functions_config"] = map[string]any{
+ "grammar_disabled": c.FunctionsConfig.GrammarConfig.NoGrammar,
+ "parallel_calls": c.FunctionsConfig.GrammarConfig.ParallelCalls,
+ "mixed_mode": c.FunctionsConfig.GrammarConfig.MixedMode,
+ "xml_format_preset": c.FunctionsConfig.XMLFormatPreset,
+ }
+ if c.Temperature != nil {
+ traceData["temperature"] = *c.Temperature
+ }
+ if c.TopP != nil {
+ traceData["top_p"] = *c.TopP
+ }
+ if c.Maxtokens != nil {
+ traceData["max_tokens"] = *c.Maxtokens
+ }
+
+ startTime := time.Now()
+ originalFn := fn
+ fn = func() (LLMResponse, error) {
+ resp, err := originalFn()
+ duration := time.Since(startTime)
+
+ traceData["response"] = resp.Response
+ traceData["token_usage"] = map[string]any{
+ "prompt": resp.Usage.Prompt,
+ "completion": resp.Usage.Completion,
+ }
+
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: duration,
+ Type: trace.BackendTraceLLM,
+ ModelName: c.Name,
+ Backend: c.Backend,
+ Summary: trace.GenerateLLMSummary(messages, s),
+ Error: errStr,
+ Data: traceData,
+ })
+
+ return resp, err
+ }
+ }
+
return fn, nil
}
diff --git a/core/backend/rerank.go b/core/backend/rerank.go
index bcfad7382..2ff10120f 100644
--- a/core/backend/rerank.go
+++ b/core/backend/rerank.go
@@ -3,8 +3,10 @@ package backend
import (
"context"
"fmt"
+ "time"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/pkg/grpc/proto"
model "github.com/mudler/LocalAI/pkg/model"
)
@@ -20,7 +22,35 @@ func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *
return nil, fmt.Errorf("could not load rerank model")
}
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
res, err := rerankModel.Rerank(context.Background(), request)
+ if appConfig.EnableTracing {
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceRerank,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(request.Query, 200),
+ Error: errStr,
+ Data: map[string]any{
+ "query": request.Query,
+ "documents_count": len(request.Documents),
+ "top_n": request.TopN,
+ },
+ })
+ }
+
return res, err
}
diff --git a/core/backend/soundgeneration.go b/core/backend/soundgeneration.go
index 60fdad231..f2b03c45c 100644
--- a/core/backend/soundgeneration.go
+++ b/core/backend/soundgeneration.go
@@ -5,8 +5,10 @@ import (
"fmt"
"os"
"path/filepath"
+ "time"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/utils"
@@ -92,7 +94,51 @@ func SoundGeneration(
req.Instrumental = instrumental
}
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
res, err := soundGenModel.SoundGeneration(context.Background(), req)
+
+ if appConfig.EnableTracing {
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ } else if res != nil && !res.Success {
+ errStr = fmt.Sprintf("sound generation error: %s", res.Message)
+ }
+
+ summary := trace.TruncateString(text, 200)
+ if summary == "" && caption != "" {
+ summary = trace.TruncateString(caption, 200)
+ }
+
+ traceData := map[string]any{
+ "text": text,
+ "caption": caption,
+ "lyrics": lyrics,
+ }
+ if duration != nil {
+ traceData["duration"] = *duration
+ }
+ if temperature != nil {
+ traceData["temperature"] = *temperature
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceSoundGeneration,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: summary,
+ Error: errStr,
+ Data: traceData,
+ })
+ }
+
if err != nil {
return "", nil, err
}
diff --git a/core/backend/tokenize.go b/core/backend/tokenize.go
index 5803e44be..60d8568f2 100644
--- a/core/backend/tokenize.go
+++ b/core/backend/tokenize.go
@@ -1,7 +1,10 @@
package backend
import (
+ "time"
+
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc"
"github.com/mudler/LocalAI/pkg/model"
@@ -21,8 +24,41 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model
predictOptions := gRPCPredictOpts(modelConfig, loader.ModelPath)
predictOptions.Prompt = s
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
// tokenize the string
resp, err := inferenceModel.TokenizeString(appConfig.Context, predictOptions)
+
+ if appConfig.EnableTracing {
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ tokenCount := 0
+ if resp.Tokens != nil {
+ tokenCount = len(resp.Tokens)
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceTokenize,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(s, 200),
+ Error: errStr,
+ Data: map[string]any{
+ "input_text": trace.TruncateString(s, 1000),
+ "token_count": tokenCount,
+ },
+ })
+ }
+
if err != nil {
return schema.TokenizeResponse{}, err
}
diff --git a/core/backend/transcript.go b/core/backend/transcript.go
index 4c721e986..dbbf718a3 100644
--- a/core/backend/transcript.go
+++ b/core/backend/transcript.go
@@ -6,6 +6,7 @@ import (
"time"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -28,6 +29,12 @@ func ModelTranscription(audio, language string, translate, diarize bool, prompt
return nil, fmt.Errorf("could not load transcription model")
}
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
r, err := transcriptionModel.AudioTranscription(context.Background(), &proto.TranscriptRequest{
Dst: audio,
Language: language,
@@ -37,6 +44,24 @@ func ModelTranscription(audio, language string, translate, diarize bool, prompt
Prompt: prompt,
})
if err != nil {
+ if appConfig.EnableTracing {
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceTranscription,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(audio, 200),
+ Error: err.Error(),
+ Data: map[string]any{
+ "audio_file": audio,
+ "language": language,
+ "translate": translate,
+ "diarize": diarize,
+ "prompt": prompt,
+ },
+ })
+ }
return nil, err
}
tr := &schema.TranscriptionResult{
@@ -57,5 +82,26 @@ func ModelTranscription(audio, language string, translate, diarize bool, prompt
Speaker: s.Speaker,
})
}
+
+ if appConfig.EnableTracing {
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceTranscription,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(audio+" -> "+tr.Text, 200),
+ Data: map[string]any{
+ "audio_file": audio,
+ "language": language,
+ "translate": translate,
+ "diarize": diarize,
+ "prompt": prompt,
+ "result_text": tr.Text,
+ "segments_count": len(tr.Segments),
+ },
+ })
+ }
+
return tr, err
}
diff --git a/core/backend/tts.go b/core/backend/tts.go
index 6e97ca8c3..7859cd67c 100644
--- a/core/backend/tts.go
+++ b/core/backend/tts.go
@@ -8,8 +8,10 @@ import (
"fmt"
"os"
"path/filepath"
+ "time"
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
laudio "github.com/mudler/LocalAI/pkg/audio"
"github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -60,6 +62,12 @@ func ModelTTS(
modelPath = modelConfig.Model // skip this step if it fails?????
}
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
res, err := ttsModel.TTS(context.Background(), &proto.TTSRequest{
Text: text,
Model: modelPath,
@@ -67,6 +75,31 @@ func ModelTTS(
Dst: filePath,
Language: &language,
})
+
+ if appConfig.EnableTracing {
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ } else if !res.Success {
+ errStr = fmt.Sprintf("TTS error: %s", res.Message)
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceTTS,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(text, 200),
+ Error: errStr,
+ Data: map[string]any{
+ "text": text,
+ "voice": voice,
+ "language": language,
+ },
+ })
+ }
+
if err != nil {
return "", nil, err
}
@@ -115,6 +148,12 @@ func ModelTTSStream(
modelPath = modelConfig.Model // skip this step if it fails?????
}
+ var startTime time.Time
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+ startTime = time.Now()
+ }
+
var sampleRate uint32 = 16000 // default
headerSent := false
var callbackErr error
@@ -171,6 +210,34 @@ func ModelTTSStream(
}
})
+ resultErr := err
+ if callbackErr != nil {
+ resultErr = callbackErr
+ }
+
+ if appConfig.EnableTracing {
+ errStr := ""
+ if resultErr != nil {
+ errStr = resultErr.Error()
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: time.Since(startTime),
+ Type: trace.BackendTraceTTS,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(text, 200),
+ Error: errStr,
+ Data: map[string]any{
+ "text": text,
+ "voice": voice,
+ "language": language,
+ "streaming": true,
+ },
+ })
+ }
+
if callbackErr != nil {
return callbackErr
}
diff --git a/core/backend/video.go b/core/backend/video.go
index 666a76252..277320515 100644
--- a/core/backend/video.go
+++ b/core/backend/video.go
@@ -1,7 +1,10 @@
package backend
import (
+ "time"
+
"github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/pkg/grpc/proto"
model "github.com/mudler/LocalAI/pkg/model"
@@ -37,5 +40,46 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
return err
}
+ if appConfig.EnableTracing {
+ trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+
+ traceData := map[string]any{
+ "prompt": prompt,
+ "negative_prompt": negativePrompt,
+ "height": height,
+ "width": width,
+ "num_frames": numFrames,
+ "fps": fps,
+ "seed": seed,
+ "cfg_scale": cfgScale,
+ "step": step,
+ }
+
+ startTime := time.Now()
+ originalFn := fn
+ fn = func() error {
+ err := originalFn()
+ duration := time.Since(startTime)
+
+ errStr := ""
+ if err != nil {
+ errStr = err.Error()
+ }
+
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: startTime,
+ Duration: duration,
+ Type: trace.BackendTraceVideoGeneration,
+ ModelName: modelConfig.Name,
+ Backend: modelConfig.Backend,
+ Summary: trace.TruncateString(prompt, 200),
+ Error: errStr,
+ Data: traceData,
+ })
+
+ return err
+ }
+ }
+
return fn, nil
}
diff --git a/core/http/routes/ui.go b/core/http/routes/ui.go
index b238c01ab..4cd9a72bd 100644
--- a/core/http/routes/ui.go
+++ b/core/http/routes/ui.go
@@ -7,6 +7,7 @@ import (
"github.com/mudler/LocalAI/core/http/endpoints/localai"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/services"
+ "github.com/mudler/LocalAI/core/trace"
"github.com/mudler/LocalAI/internal"
"github.com/mudler/LocalAI/pkg/model"
)
@@ -430,4 +431,13 @@ func RegisterUIRoutes(app *echo.Echo,
return c.NoContent(204)
})
+ app.GET("/api/backend-traces", func(c echo.Context) error {
+ return c.JSON(200, trace.GetBackendTraces())
+ })
+
+ app.POST("/api/backend-traces/clear", func(c echo.Context) error {
+ trace.ClearBackendTraces()
+ return c.NoContent(204)
+ })
+
}
diff --git a/core/http/views/traces.html b/core/http/views/traces.html
index 667643e45..5d4f120e1 100644
--- a/core/http/views/traces.html
+++ b/core/http/views/traces.html
@@ -5,7 +5,7 @@
{{template "views/partials/navbar" .}}
-
+
@@ -40,10 +40,10 @@
- API Traces
+ Traces
-
View logged API requests and responses
-
+
View logged API requests, responses, and backend operations
+
+
+
+
+
+
+
+
Tracing Settings
-
Configure API tracing
+
Configure API and backend tracing
@@ -103,8 +129,8 @@
-
-
+
+
+
+ No API traces recorded yet.
+
-
+
+
+
+
+
+
+ | Type |
+ Timestamp |
+ Model |
+ Summary |
+ Duration |
+ Status |
+ Actions |
+
+
+
+
+
+ |
+
+ |
+ |
+ |
+ |
+ |
+
+
+
+
+
+
+
+ |
+
+
+ |
+
+
+
+
+
+ No backend traces recorded yet.
+
+
+
+
+
-
Trace Details
+ API Trace Details
@@ -155,6 +234,96 @@
+
+
+
+
+
+
+
Backend Trace Details
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ | Field |
+ Value |
+
+
+
+
+
+ |
+
+
+
+
+
+
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -176,21 +345,44 @@