mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-04-29 03:24:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore(refactor): move logging to common package based on slog (#7668)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-12-21 19:33:13 +01:00
committed by GitHub
parent 38cde81ff4
commit c37785b78c
88 changed files with 1250 additions and 1320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/go/local-store/debug.go
View File

@@ -4,11 +4,11 @@
package main package main
import ( import (
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func assert(cond bool, msg string) { func assert(cond bool, msg string) {
if !cond { if !cond {
log.Fatal().Stack().Msg(msg) xlog.Fatal().Stack().Msg(msg)
} }
} }

6
backend/go/local-store/main.go
View File

@@ -4,11 +4,9 @@ package main
import ( import (
"flag" "flag"
"os"
grpc "github.com/mudler/LocalAI/pkg/grpc" grpc "github.com/mudler/LocalAI/pkg/grpc"
"github.com/rs/zerolog" "github.com/mudler/xlog"
"github.com/rs/zerolog/log"
) )
var ( var (
@@ -16,7 +14,7 @@ var (
) )
func main() { func main() {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
flag.Parse() flag.Parse()

14
backend/go/local-store/store.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/grpc/base" "github.com/mudler/LocalAI/pkg/grpc/base"
pb "github.com/mudler/LocalAI/pkg/grpc/proto" pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type Store struct { type Store struct {
@@ -135,7 +135,7 @@ func (s *Store) StoresSet(opts *pb.StoresSetOptions) error {
} else { } else {
sample = k.Floats sample = k.Floats
} }
log.Debug().Msgf("Key is not normalized: %v", sample) xlog.Debug("Key is not normalized", "sample", sample)
} }
kvs[i] = Pair{ kvs[i] = Pair{
@@ -238,7 +238,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k)) assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k))
} }
log.Debug().Msgf("Delete: found = %v, t = %d, j = %d, len(merge_ks) = %d, len(merge_vs) = %d", found, len(tail_ks), j, len(merge_ks), len(merge_vs)) xlog.Debug("Delete", "found", found, "tailLen", len(tail_ks), "j", j, "mergeKeysLen", len(merge_ks), "mergeValuesLen", len(merge_vs))
} }
merge_ks = append(merge_ks, tail_ks...) merge_ks = append(merge_ks, tail_ks...)
@@ -261,7 +261,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
}(), "Keys to delete still present") }(), "Keys to delete still present")
if len(s.keys) != l { if len(s.keys) != l {
log.Debug().Msgf("Delete: Some keys not found: len(s.keys) = %d, l = %d", len(s.keys), l) xlog.Debug("Delete: Some keys not found", "keysLen", len(s.keys), "expectedLen", l)
} }
return nil return nil
@@ -273,7 +273,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
ks := sortIntoKeySlicese(opts.Keys) ks := sortIntoKeySlicese(opts.Keys)
if len(s.keys) == 0 { if len(s.keys) == 0 {
log.Debug().Msgf("Get: No keys in store") xlog.Debug("Get: No keys in store")
} }
if s.keyLen == -1 { if s.keyLen == -1 {
@@ -305,7 +305,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
} }
if len(pbKeys) != len(opts.Keys) { if len(pbKeys) != len(opts.Keys) {
log.Debug().Msgf("Get: Some keys not found: len(pbKeys) = %d, len(opts.Keys) = %d, len(s.Keys) = %d", len(pbKeys), len(opts.Keys), len(s.keys)) xlog.Debug("Get: Some keys not found", "pbKeysLen", len(pbKeys), "optsKeysLen", len(opts.Keys), "storeKeysLen", len(s.keys))
} }
return pb.StoresGetResult{ return pb.StoresGetResult{
@@ -507,7 +507,7 @@ func (s *Store) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, err
} else { } else {
sample = tk sample = tk
} }
log.Debug().Msgf("Trying to compare non-normalized key with normalized keys: %v", sample) xlog.Debug("Trying to compare non-normalized key with normalized keys", "sample", sample)
} }
return s.StoresFindFallback(opts) return s.StoresFindFallback(opts)

40
cmd/local-ai/main.go
View File

@@ -8,10 +8,7 @@ import (
"github.com/joho/godotenv" "github.com/joho/godotenv"
"github.com/mudler/LocalAI/core/cli" "github.com/mudler/LocalAI/core/cli"
"github.com/mudler/LocalAI/internal" "github.com/mudler/LocalAI/internal"
"github.com/mudler/cogito/pkg/xlog" "github.com/mudler/xlog"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
_ "github.com/mudler/LocalAI/swagger" _ "github.com/mudler/LocalAI/swagger"
) )
@@ -19,9 +16,8 @@ import (
func main() { func main() {
var err error var err error
// Initialize zerolog at a level of INFO, we will set the desired level after we parse the CLI options // Initialize xlog at a level of INFO, we will set the desired level after we parse the CLI options
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
zerolog.SetGlobalLevel(zerolog.InfoLevel)
// handle loading environment variables from .env files // handle loading environment variables from .env files
envFiles := []string{".env", "localai.env"} envFiles := []string{".env", "localai.env"}
@@ -33,10 +29,10 @@ func main() {
for _, envFile := range envFiles { for _, envFile := range envFiles {
if _, err := os.Stat(envFile); err == nil { if _, err := os.Stat(envFile); err == nil {
log.Debug().Str("envFile", envFile).Msg("env file found, loading environment variables from file") xlog.Debug("env file found, loading environment variables from file", "envFile", envFile)
err = godotenv.Load(envFile) err = godotenv.Load(envFile)
if err != nil { if err != nil {
log.Error().Err(err).Str("envFile", envFile).Msg("failed to load environment variables from file") xlog.Error("failed to load environment variables from file", "error", err, "envFile", envFile)
continue continue
} }
} }
@@ -68,7 +64,6 @@ Version: ${version}
logLevel := "info" logLevel := "info"
if cli.CLI.Debug && cli.CLI.LogLevel == nil { if cli.CLI.Debug && cli.CLI.LogLevel == nil {
logLevel = "debug" logLevel = "debug"
zerolog.SetGlobalLevel(zerolog.DebugLevel)
cli.CLI.LogLevel = &logLevel cli.CLI.LogLevel = &logLevel
} }
@@ -76,31 +71,12 @@ Version: ${version}
cli.CLI.LogLevel = &logLevel cli.CLI.LogLevel = &logLevel
} }
// Set cogito logger to the same level as our logger // Set xlog logger with the desired level and text format
// Leave an empty format type xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), "text"))
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), ""))
switch *cli.CLI.LogLevel {
case "error":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
log.Debug().Msg("Setting logging to error")
case "warn":
zerolog.SetGlobalLevel(zerolog.WarnLevel)
log.Debug().Msg("Setting logging to warn")
case "info":
zerolog.SetGlobalLevel(zerolog.InfoLevel)
log.Debug().Msg("Setting logging to info")
case "debug":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
log.Debug().Msg("Setting logging to debug")
case "trace":
zerolog.SetGlobalLevel(zerolog.TraceLevel)
log.Debug().Msg("Setting logging to trace")
}
// Run the thing! // Run the thing!
err = ctx.Run(&cli.CLI.Context) err = ctx.Run(&cli.CLI.Context)
if err != nil { if err != nil {
log.Fatal().Err(err).Msg("Error running the application") xlog.Fatal("Error running the application", "error", err)
} }
} }

9
core/application/agent_jobs.go
View File

@@ -4,7 +4,7 @@ import (
"time" "time"
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// RestartAgentJobService restarts the agent job service with current ApplicationConfig settings // RestartAgentJobService restarts the agent job service with current ApplicationConfig settings
@@ -15,7 +15,7 @@ func (a *Application) RestartAgentJobService() error {
// Stop existing service if running // Stop existing service if running
if a.agentJobService != nil { if a.agentJobService != nil {
if err := a.agentJobService.Stop(); err != nil { if err := a.agentJobService.Stop(); err != nil {
log.Warn().Err(err).Msg("Error stopping agent job service") xlog.Warn("Error stopping agent job service", "error", err)
} }
// Wait a bit for shutdown to complete // Wait a bit for shutdown to complete
time.Sleep(200 * time.Millisecond) time.Sleep(200 * time.Millisecond)
@@ -32,12 +32,11 @@ func (a *Application) RestartAgentJobService() error {
// Start the service // Start the service
err := agentJobService.Start(a.ApplicationConfig().Context) err := agentJobService.Start(a.ApplicationConfig().Context)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to start agent job service") xlog.Error("Failed to start agent job service", "error", err)
return err return err
} }
a.agentJobService = agentJobService a.agentJobService = agentJobService
log.Info().Msg("Agent job service restarted") xlog.Info("Agent job service restarted")
return nil return nil
} }

41
core/application/config_file_watcher.go
View File

@@ -11,7 +11,7 @@ import (
"dario.cat/mergo" "dario.cat/mergo"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
@@ -33,15 +33,15 @@ func newConfigFileHandler(appConfig *config.ApplicationConfig) configFileHandler
} }
err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true) err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
if err != nil { if err != nil {
log.Error().Err(err).Str("file", "api_keys.json").Msg("unable to register config file handler") xlog.Error("unable to register config file handler", "error", err, "file", "api_keys.json")
} }
err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true) err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
if err != nil { if err != nil {
log.Error().Err(err).Str("file", "external_backends.json").Msg("unable to register config file handler") xlog.Error("unable to register config file handler", "error", err, "file", "external_backends.json")
} }
err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true) err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true)
if err != nil { if err != nil {
log.Error().Err(err).Str("file", "runtime_settings.json").Msg("unable to register config file handler") xlog.Error("unable to register config file handler", "error", err, "file", "runtime_settings.json")
} }
// Note: agent_tasks.json and agent_jobs.json are handled by AgentJobService directly // Note: agent_tasks.json and agent_jobs.json are handled by AgentJobService directly
// The service watches and reloads these files internally // The service watches and reloads these files internally
@@ -62,14 +62,14 @@ func (c *configFileHandler) Register(filename string, handler fileHandler, runNo
func (c *configFileHandler) callHandler(filename string, handler fileHandler) { func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename)) rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
log.Trace().Str("filename", rootedFilePath).Msg("reading file for dynamic config update") xlog.Debug("reading file for dynamic config update", "filename", rootedFilePath)
fileContent, err := os.ReadFile(rootedFilePath) fileContent, err := os.ReadFile(rootedFilePath)
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
log.Error().Err(err).Str("filename", rootedFilePath).Msg("could not read file") xlog.Error("could not read file", "error", err, "filename", rootedFilePath)
} }
if err = handler(fileContent, c.appConfig); err != nil { if err = handler(fileContent, c.appConfig); err != nil {
log.Error().Err(err).Msg("WatchConfigDirectory goroutine failed to update options") xlog.Error("WatchConfigDirectory goroutine failed to update options", "error", err)
} }
} }
@@ -81,13 +81,13 @@ func (c *configFileHandler) Watch() error {
} }
if c.appConfig.DynamicConfigsDirPollInterval > 0 { if c.appConfig.DynamicConfigsDirPollInterval > 0 {
log.Debug().Msg("Poll interval set, falling back to polling for configuration changes") xlog.Debug("Poll interval set, falling back to polling for configuration changes")
ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval) ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
go func() { go func() {
for { for {
<-ticker.C <-ticker.C
for file, handler := range c.handlers { for file, handler := range c.handlers {
log.Debug().Str("file", file).Msg("polling config file") xlog.Debug("polling config file", "file", file)
c.callHandler(file, handler) c.callHandler(file, handler)
} }
} }
@@ -111,7 +111,7 @@ func (c *configFileHandler) Watch() error {
c.callHandler(filepath.Base(event.Name), handler) c.callHandler(filepath.Base(event.Name), handler)
} }
case err, ok := <-c.watcher.Errors: case err, ok := <-c.watcher.Errors:
log.Error().Err(err).Msg("config watcher error received") xlog.Error("config watcher error received", "error", err)
if !ok { if !ok {
return return
} }
@@ -135,8 +135,7 @@ func (c *configFileHandler) Stop() error {
func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler { func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error { handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing api keys runtime update") xlog.Debug("processing api keys runtime update", "numKeys", len(startupAppConfig.ApiKeys))
log.Trace().Int("numKeys", len(startupAppConfig.ApiKeys)).Msg("api keys provided at startup")
if len(fileContent) > 0 { if len(fileContent) > 0 {
// Parse JSON content from the file // Parse JSON content from the file
@@ -146,14 +145,14 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
return err return err
} }
log.Trace().Int("numKeys", len(fileKeys)).Msg("discovered API keys from api keys dynamic config dile") xlog.Debug("discovered API keys from api keys dynamic config file", "numKeys", len(fileKeys))
appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...) appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
} else { } else {
log.Trace().Msg("no API keys discovered from dynamic config file") xlog.Debug("no API keys discovered from dynamic config file")
appConfig.ApiKeys = startupAppConfig.ApiKeys appConfig.ApiKeys = startupAppConfig.ApiKeys
} }
log.Trace().Int("numKeys", len(appConfig.ApiKeys)).Msg("total api keys after processing") xlog.Debug("total api keys after processing", "numKeys", len(appConfig.ApiKeys))
return nil return nil
} }
@@ -162,7 +161,7 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler { func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error { handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing external_backends.json") xlog.Debug("processing external_backends.json")
if len(fileContent) > 0 { if len(fileContent) > 0 {
// Parse JSON content from the file // Parse JSON content from the file
@@ -179,7 +178,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
} else { } else {
appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
} }
log.Debug().Msg("external backends loaded from external_backends.json") xlog.Debug("external backends loaded from external_backends.json")
return nil return nil
} }
return handler return handler
@@ -187,7 +186,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler { func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error { handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing runtime_settings.json") xlog.Debug("processing runtime_settings.json")
// Determine if settings came from env vars by comparing with startup config // Determine if settings came from env vars by comparing with startup config
// startupAppConfig contains the original values set from env vars at startup. // startupAppConfig contains the original values set from env vars at startup.
@@ -241,7 +240,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
if err == nil { if err == nil {
appConfig.WatchDogIdleTimeout = dur appConfig.WatchDogIdleTimeout = dur
} else { } else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json") xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
} }
} }
if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout { if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout {
@@ -249,7 +248,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
if err == nil { if err == nil {
appConfig.WatchDogBusyTimeout = dur appConfig.WatchDogBusyTimeout = dur
} else { } else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json") xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
} }
} }
// Handle MaxActiveBackends (new) and SingleBackend (deprecated) // Handle MaxActiveBackends (new) and SingleBackend (deprecated)
@@ -340,7 +339,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
} }
} }
} }
log.Debug().Msg("runtime settings loaded from runtime_settings.json") xlog.Debug("runtime settings loaded from runtime_settings.json")
return nil return nil
} }
return handler return handler

31
core/application/p2p.go
View File

@@ -14,8 +14,7 @@ import (
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/mudler/edgevpn/pkg/node" "github.com/mudler/edgevpn/pkg/node"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
zlog "github.com/rs/zerolog/log"
) )
func (a *Application) StopP2P() error { func (a *Application) StopP2P() error {
@@ -86,14 +85,14 @@ func (a *Application) StartP2P() error {
} }
// Attach a ServiceDiscoverer to the p2p node // Attach a ServiceDiscoverer to the p2p node
log.Info().Msg("Starting P2P server discovery...") xlog.Info("Starting P2P server discovery...")
if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) { if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
var tunnelAddresses []string var tunnelAddresses []string
for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) { for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
if v.IsOnline() { if v.IsOnline() {
tunnelAddresses = append(tunnelAddresses, v.TunnelAddress) tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
} else { } else {
log.Info().Msgf("Node %s is offline", v.ID) xlog.Info("Node is offline", "node", v.ID)
} }
} }
if a.applicationConfig.TunnelCallback != nil { if a.applicationConfig.TunnelCallback != nil {
@@ -143,17 +142,17 @@ func (a *Application) RestartP2P() error {
// Start P2P stack in a goroutine // Start P2P stack in a goroutine
go func() { go func() {
if err := a.StartP2P(); err != nil { if err := a.StartP2P(); err != nil {
log.Error().Err(err).Msg("Failed to start P2P stack") xlog.Error("Failed to start P2P stack", "error", err)
cancel() // Cancel context on error cancel() // Cancel context on error
} }
}() }()
log.Info().Msg("P2P stack restarted with new settings") xlog.Info("P2P stack restarted with new settings")
return nil return nil
} }
func syncState(ctx context.Context, n *node.Node, app *Application) error { func syncState(ctx context.Context, n *node.Node, app *Application) error {
zlog.Debug().Msg("[p2p-sync] Syncing state") xlog.Debug("[p2p-sync] Syncing state")
whatWeHave := []string{} whatWeHave := []string{}
for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() { for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
@@ -162,20 +161,20 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
ledger, _ := n.Ledger() ledger, _ := n.Ledger()
currentData := ledger.CurrentData() currentData := ledger.CurrentData()
zlog.Debug().Msgf("[p2p-sync] Current data: %v", currentData) xlog.Debug("[p2p-sync] Current data", "data", currentData)
data, exists := ledger.GetKey("shared_state", "models") data, exists := ledger.GetKey("shared_state", "models")
if !exists { if !exists {
ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave) ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
zlog.Debug().Msgf("No models found in the ledger, announced our models: %v", whatWeHave) xlog.Debug("No models found in the ledger, announced our models", "models", whatWeHave)
} }
models := []string{} models := []string{}
if err := data.Unmarshal(&models); err != nil { if err := data.Unmarshal(&models); err != nil {
zlog.Warn().Err(err).Msg("error unmarshalling models") xlog.Warn("error unmarshalling models", "error", err)
return nil return nil
} }
zlog.Debug().Msgf("[p2p-sync] Models that are present in this instance: %v\nModels that are in the ledger: %v", whatWeHave, models) xlog.Debug("[p2p-sync] Models comparison", "ourModels", whatWeHave, "ledgerModels", models)
// Sync with our state // Sync with our state
whatIsNotThere := []string{} whatIsNotThere := []string{}
@@ -185,7 +184,7 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
} }
} }
if len(whatIsNotThere) > 0 { if len(whatIsNotThere) > 0 {
zlog.Debug().Msgf("[p2p-sync] Announcing our models: %v", append(models, whatIsNotThere...)) xlog.Debug("[p2p-sync] Announcing our models", "models", append(models, whatIsNotThere...))
ledger.AnnounceUpdate( ledger.AnnounceUpdate(
ctx, ctx,
1*time.Minute, 1*time.Minute,
@@ -198,16 +197,16 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
// Check if we have a model that is not in our state, otherwise install it // Check if we have a model that is not in our state, otherwise install it
for _, model := range models { for _, model := range models {
if slices.Contains(whatWeHave, model) { if slices.Contains(whatWeHave, model) {
zlog.Debug().Msgf("[p2p-sync] Model %s is already present in this instance", model) xlog.Debug("[p2p-sync] Model is already present in this instance", "model", model)
continue continue
} }
// we install model // we install model
zlog.Info().Msgf("[p2p-sync] Installing model which is not present in this instance: %s", model) xlog.Info("[p2p-sync] Installing model which is not present in this instance", "model", model)
uuid, err := uuid.NewUUID() uuid, err := uuid.NewUUID()
if err != nil { if err != nil {
zlog.Error().Err(err).Msg("error generating UUID") xlog.Error("error generating UUID", "error", err)
continue continue
} }
@@ -230,7 +229,7 @@ func (a *Application) p2pSync(ctx context.Context, n *node.Node) error {
return return
case <-time.After(1 * time.Minute): case <-time.After(1 * time.Minute):
if err := syncState(ctx, n, a); err != nil { if err := syncState(ctx, n, a); err != nil {
zlog.Error().Err(err).Msg("error syncing state") xlog.Error("error syncing state", "error", err)
} }
} }

56
core/application/startup.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func New(opts ...config.AppOption) (*Application, error) { func New(opts ...config.AppOption) (*Application, error) {
@@ -28,8 +28,8 @@ func New(opts ...config.AppOption) (*Application, error) {
application := newApplication(options) application := newApplication(options)
application.startupConfig = &startupConfigCopy application.startupConfig = &startupConfigCopy
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.SystemState.Model.ModelsPath) xlog.Info("Starting LocalAI", "threads", options.Threads, "modelsPath", options.SystemState.Model.ModelsPath)
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion()) xlog.Info("LocalAI version", "version", internal.PrintableVersion())
if err := application.start(); err != nil { if err := application.start(); err != nil {
return nil, err return nil, err
@@ -37,14 +37,14 @@ func New(opts ...config.AppOption) (*Application, error) {
caps, err := xsysinfo.CPUCapabilities() caps, err := xsysinfo.CPUCapabilities()
if err == nil { if err == nil {
log.Debug().Msgf("CPU capabilities: %v", caps) xlog.Debug("CPU capabilities", "capabilities", caps)
} }
gpus, err := xsysinfo.GPUs() gpus, err := xsysinfo.GPUs()
if err == nil { if err == nil {
log.Debug().Msgf("GPU count: %d", len(gpus)) xlog.Debug("GPU count", "count", len(gpus))
for _, gpu := range gpus { for _, gpu := range gpus {
log.Debug().Msgf("GPU: %s", gpu.String()) xlog.Debug("GPU", "gpu", gpu.String())
} }
} }
@@ -71,33 +71,33 @@ func New(opts ...config.AppOption) (*Application, error) {
} }
if err := coreStartup.InstallModels(options.Context, application.GalleryService(), options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil { if err := coreStartup.InstallModels(options.Context, application.GalleryService(), options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
log.Error().Err(err).Msg("error installing models") xlog.Error("error installing models", "error", err)
} }
for _, backend := range options.ExternalBackends { for _, backend := range options.ExternalBackends {
if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil { if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
log.Error().Err(err).Msg("error installing external backend") xlog.Error("error installing external backend", "error", err)
} }
} }
configLoaderOpts := options.ToConfigLoaderOptions() configLoaderOpts := options.ToConfigLoaderOptions()
if err := application.ModelConfigLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil { if err := application.ModelConfigLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil {
log.Error().Err(err).Msg("error loading config files") xlog.Error("error loading config files", "error", err)
} }
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil { if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
log.Error().Err(err).Msg("error registering external backends") xlog.Error("error registering external backends", "error", err)
} }
if options.ConfigFile != "" { if options.ConfigFile != "" {
if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
log.Error().Err(err).Msg("error loading config file") xlog.Error("error loading config file", "error", err)
} }
} }
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil { if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
log.Error().Err(err).Msg("error downloading models") xlog.Error("error downloading models", "error", err)
} }
if options.PreloadJSONModels != "" { if options.PreloadJSONModels != "" {
@@ -114,7 +114,7 @@ func New(opts ...config.AppOption) (*Application, error) {
if options.Debug { if options.Debug {
for _, v := range application.ModelConfigLoader().GetAllModelsConfigs() { for _, v := range application.ModelConfigLoader().GetAllModelsConfigs() {
log.Debug().Msgf("Model: %s (config: %+v)", v.Name, v) xlog.Debug("Model", "name", v.Name, "config", v)
} }
} }
@@ -128,10 +128,10 @@ func New(opts ...config.AppOption) (*Application, error) {
// turn off any process that was started by GRPC if the context is canceled // turn off any process that was started by GRPC if the context is canceled
go func() { go func() {
<-options.Context.Done() <-options.Context.Done()
log.Debug().Msgf("Context canceled, shutting down") xlog.Debug("Context canceled, shutting down")
err := application.ModelLoader().StopAllGRPC() err := application.ModelLoader().StopAllGRPC()
if err != nil { if err != nil {
log.Error().Err(err).Msg("error while stopping all grpc backends") xlog.Error("error while stopping all grpc backends", "error", err)
} }
}() }()
@@ -145,7 +145,7 @@ func New(opts ...config.AppOption) (*Application, error) {
return nil, err return nil, err
} }
log.Debug().Msgf("Auto loading model %s into memory from file: %s", m, cfg.Model) xlog.Debug("Auto loading model into memory from file", "model", m, "file", cfg.Model)
o := backend.ModelOptions(*cfg, options) o := backend.ModelOptions(*cfg, options)
@@ -160,7 +160,7 @@ func New(opts ...config.AppOption) (*Application, error) {
// Watch the configuration directory // Watch the configuration directory
startWatcher(options) startWatcher(options)
log.Info().Msg("core/startup process completed!") xlog.Info("core/startup process completed!")
return application, nil return application, nil
} }
@@ -174,18 +174,18 @@ func startWatcher(options *config.ApplicationConfig) {
if os.IsNotExist(err) { if os.IsNotExist(err) {
// We try to create the directory if it does not exist and was specified // We try to create the directory if it does not exist and was specified
if err := os.MkdirAll(options.DynamicConfigsDir, 0700); err != nil { if err := os.MkdirAll(options.DynamicConfigsDir, 0700); err != nil {
log.Error().Err(err).Msg("failed creating DynamicConfigsDir") xlog.Error("failed creating DynamicConfigsDir", "error", err)
} }
} else { } else {
// something else happened, we log the error and don't start the watcher // something else happened, we log the error and don't start the watcher
log.Error().Err(err).Msg("failed to read DynamicConfigsDir, watcher will not be started") xlog.Error("failed to read DynamicConfigsDir, watcher will not be started", "error", err)
return return
} }
} }
configHandler := newConfigFileHandler(options) configHandler := newConfigFileHandler(options)
if err := configHandler.Watch(); err != nil { if err := configHandler.Watch(); err != nil {
log.Error().Err(err).Msg("failed creating watcher") xlog.Error("failed creating watcher", "error", err)
} }
} }
@@ -211,17 +211,17 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
fileContent, err := os.ReadFile(settingsFile) fileContent, err := os.ReadFile(settingsFile)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
log.Debug().Msg("runtime_settings.json not found, using defaults") xlog.Debug("runtime_settings.json not found, using defaults")
return return
} }
log.Warn().Err(err).Msg("failed to read runtime_settings.json") xlog.Warn("failed to read runtime_settings.json", "error", err)
return return
} }
var settings config.RuntimeSettings var settings config.RuntimeSettings
if err := json.Unmarshal(fileContent, &settings); err != nil { if err := json.Unmarshal(fileContent, &settings); err != nil {
log.Warn().Err(err).Msg("failed to parse runtime_settings.json") xlog.Warn("failed to parse runtime_settings.json", "error", err)
return return
} }
@@ -257,7 +257,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil { if err == nil {
options.WatchDogIdleTimeout = dur options.WatchDogIdleTimeout = dur
} else { } else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json") xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
} }
} }
} }
@@ -267,7 +267,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil { if err == nil {
options.WatchDogBusyTimeout = dur options.WatchDogBusyTimeout = dur
} else { } else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json") xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
} }
} }
} }
@@ -277,7 +277,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil { if err == nil {
options.WatchDogInterval = dur options.WatchDogInterval = dur
} else { } else {
log.Warn().Err(err).Str("interval", *settings.WatchdogInterval).Msg("invalid watchdog interval in runtime_settings.json") xlog.Warn("invalid watchdog interval in runtime_settings.json", "error", err, "interval", *settings.WatchdogInterval)
options.WatchDogInterval = model.DefaultWatchdogInterval options.WatchDogInterval = model.DefaultWatchdogInterval
} }
} }
@@ -331,7 +331,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
} }
} }
log.Debug().Msg("Runtime settings loaded from runtime_settings.json") xlog.Debug("Runtime settings loaded from runtime_settings.json")
} }
// initializeWatchdog initializes the watchdog with current ApplicationConfig settings // initializeWatchdog initializes the watchdog with current ApplicationConfig settings
@@ -362,7 +362,7 @@ func initializeWatchdog(application *Application, options *config.ApplicationCon
go func() { go func() {
<-options.Context.Done() <-options.Context.Done()
log.Debug().Msgf("Context canceled, shutting down") xlog.Debug("Context canceled, shutting down")
wd.Shutdown() wd.Shutdown()
}() }()
} }

17
core/application/watchdog.go
View File

@@ -4,7 +4,7 @@ import (
"time" "time"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func (a *Application) StopWatchdog() error { func (a *Application) StopWatchdog() error {
@@ -52,24 +52,17 @@ func (a *Application) startWatchdog() error {
go func() { go func() {
select { select {
case <-a.watchdogStop: case <-a.watchdogStop:
log.Debug().Msg("Watchdog stop signal received") xlog.Debug("Watchdog stop signal received")
wd.Shutdown() wd.Shutdown()
case <-appConfig.Context.Done(): case <-appConfig.Context.Done():
log.Debug().Msg("Context canceled, shutting down watchdog") xlog.Debug("Context canceled, shutting down watchdog")
wd.Shutdown() wd.Shutdown()
} }
}() }()
log.Info(). xlog.Info("Watchdog started with new settings", "lruLimit", lruLimit, "busyCheck", appConfig.WatchDogBusy, "idleCheck", appConfig.WatchDogIdle, "memoryReclaimer", appConfig.MemoryReclaimerEnabled, "memoryThreshold", appConfig.MemoryReclaimerThreshold, "interval", appConfig.WatchDogInterval)
Int("lruLimit", lruLimit).
Bool("busyCheck", appConfig.WatchDogBusy).
Bool("idleCheck", appConfig.WatchDogIdle).
Bool("memoryReclaimer", appConfig.MemoryReclaimerEnabled).
Float64("memoryThreshold", appConfig.MemoryReclaimerThreshold).
Dur("interval", appConfig.WatchDogInterval).
Msg("Watchdog started with new settings")
} else { } else {
log.Info().Msg("Watchdog disabled") xlog.Info("Watchdog disabled")
} }
return nil return nil

8
core/backend/llm.go
View File

@@ -9,7 +9,7 @@ import (
"sync" "sync"
"unicode/utf8" "unicode/utf8"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
@@ -49,7 +49,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
// if we failed to load the model, we try to download it // if we failed to load the model, we try to download it
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries) err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile) xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
//return nil, err //return nil, err
} }
} }
@@ -225,7 +225,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
if !ok { if !ok {
r, err := regexp.Compile(c) r, err := regexp.Compile(c)
if err != nil { if err != nil {
log.Fatal().Err(err).Msg("failed to compile regex") xlog.Fatal("failed to compile regex", "error", err)
} }
cutstrings[c] = r cutstrings[c] = r
reg = cutstrings[c] reg = cutstrings[c]
@@ -242,7 +242,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
if !ok { if !ok {
regex, err := regexp.Compile(r) regex, err := regexp.Compile(r)
if err != nil { if err != nil {
log.Fatal().Err(err).Msg("failed to compile regex") xlog.Fatal("failed to compile regex", "error", err)
} }
cutstrings[r] = regex cutstrings[r] = regex
reg = regex reg = regex

4
core/backend/options.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
pb "github.com/mudler/LocalAI/pkg/grpc/proto" pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option { func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
@@ -208,7 +208,7 @@ func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions
if err == nil { if err == nil {
promptCachePath = p promptCachePath = p
} else { } else {
log.Error().Err(err).Str("promptCachePath", promptCachePath).Msg("error creating prompt cache folder") xlog.Error("error creating prompt cache folder", "error", err, "promptCachePath", promptCachePath)
} }
} }

10
core/cli/backends.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/schollz/progressbar/v3" "github.com/schollz/progressbar/v3"
) )
@@ -49,7 +49,7 @@ type BackendsCMD struct {
func (bl *BackendsList) Run(ctx *cliContext.Context) error { func (bl *BackendsList) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery var galleries []config.Gallery
if err := json.Unmarshal([]byte(bl.BackendGalleries), &galleries); err != nil { if err := json.Unmarshal([]byte(bl.BackendGalleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries") xlog.Error("unable to load galleries", "error", err)
} }
systemState, err := system.GetSystemState( systemState, err := system.GetSystemState(
@@ -77,7 +77,7 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error {
func (bi *BackendsInstall) Run(ctx *cliContext.Context) error { func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery var galleries []config.Gallery
if err := json.Unmarshal([]byte(bi.BackendGalleries), &galleries); err != nil { if err := json.Unmarshal([]byte(bi.BackendGalleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries") xlog.Error("unable to load galleries", "error", err)
} }
systemState, err := system.GetSystemState( systemState, err := system.GetSystemState(
@@ -98,7 +98,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
v := int(percentage * 10) v := int(percentage * 10)
err := progressBar.Set(v) err := progressBar.Set(v)
if err != nil { if err != nil {
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar") xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
} }
} }
@@ -113,7 +113,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error { func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
for _, backendName := range bu.BackendArgs { for _, backendName := range bu.BackendArgs {
log.Info().Str("backend", backendName).Msg("uninstalling backend") xlog.Info("uninstalling backend", "backend", backendName)
systemState, err := system.GetSystemState( systemState, err := system.GetSystemState(
system.WithBackendSystemPath(bu.BackendsSystemPath), system.WithBackendSystemPath(bu.BackendsSystemPath),

4
core/cli/explorer.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/explorer" "github.com/mudler/LocalAI/core/explorer"
"github.com/mudler/LocalAI/core/http" "github.com/mudler/LocalAI/core/http"
"github.com/mudler/LocalAI/pkg/signals" "github.com/mudler/LocalAI/pkg/signals"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type ExplorerCMD struct { type ExplorerCMD struct {
@@ -51,7 +51,7 @@ func (e *ExplorerCMD) Run(ctx *cliContext.Context) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel() defer cancel()
if err := appHTTP.Shutdown(ctx); err != nil { if err := appHTTP.Shutdown(ctx); err != nil {
log.Error().Err(err).Msg("error during shutdown") xlog.Error("error during shutdown", "error", err)
} }
}) })

14
core/cli/models.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/schollz/progressbar/v3" "github.com/schollz/progressbar/v3"
) )
@@ -46,7 +46,7 @@ type ModelsCMD struct {
func (ml *ModelsList) Run(ctx *cliContext.Context) error { func (ml *ModelsList) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery var galleries []config.Gallery
if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil { if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries") xlog.Error("unable to load galleries", "error", err)
} }
systemState, err := system.GetSystemState( systemState, err := system.GetSystemState(
@@ -88,12 +88,12 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery var galleries []config.Gallery
if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil { if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries") xlog.Error("unable to load galleries", "error", err)
} }
var backendGalleries []config.Gallery var backendGalleries []config.Gallery
if err := json.Unmarshal([]byte(mi.BackendGalleries), &backendGalleries); err != nil { if err := json.Unmarshal([]byte(mi.BackendGalleries), &backendGalleries); err != nil {
log.Error().Err(err).Msg("unable to load backend galleries") xlog.Error("unable to load backend galleries", "error", err)
} }
for _, modelName := range mi.ModelArgs { for _, modelName := range mi.ModelArgs {
@@ -108,7 +108,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
v := int(percentage * 10) v := int(percentage * 10)
err := progressBar.Set(v) err := progressBar.Set(v)
if err != nil { if err != nil {
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar") xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
} }
} }
//startup.InstallModels() //startup.InstallModels()
@@ -122,7 +122,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
if !modelURI.LooksLikeOCI() { if !modelURI.LooksLikeOCI() {
model := gallery.FindGalleryElement(models, modelName) model := gallery.FindGalleryElement(models, modelName)
if model == nil { if model == nil {
log.Error().Str("model", modelName).Msg("model not found") xlog.Error("model not found", "model", modelName)
return err return err
} }
@@ -131,7 +131,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
return err return err
} }
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model") xlog.Info("installing model", "model", modelName, "license", model.License)
} }
modelLoader := model.NewModelLoader(systemState) modelLoader := model.NewModelLoader(systemState)

19
core/cli/run.go
View File

@@ -15,8 +15,7 @@ import (
"github.com/mudler/LocalAI/internal" "github.com/mudler/LocalAI/internal"
"github.com/mudler/LocalAI/pkg/signals" "github.com/mudler/LocalAI/pkg/signals"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog" "github.com/mudler/xlog"
"github.com/rs/zerolog/log"
) )
type RunCMD struct { type RunCMD struct {
@@ -108,7 +107,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
config.WithYAMLConfigPreload(r.PreloadModelsConfig), config.WithYAMLConfigPreload(r.PreloadModelsConfig),
config.WithSystemState(systemState), config.WithSystemState(systemState),
config.WithContextSize(r.ContextSize), config.WithContextSize(r.ContextSize),
config.WithDebug(zerolog.GlobalLevel() <= zerolog.DebugLevel), config.WithDebug(ctx.Debug || (ctx.LogLevel != nil && *ctx.LogLevel == "debug")),
config.WithGeneratedContentDir(r.GeneratedContentPath), config.WithGeneratedContentDir(r.GeneratedContentPath),
config.WithUploadDir(r.UploadPath), config.WithUploadDir(r.UploadPath),
config.WithDynamicConfigDir(r.LocalaiConfigDir), config.WithDynamicConfigDir(r.LocalaiConfigDir),
@@ -138,7 +137,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
tunnelEnvVar := strings.Join(tunnels, ",") tunnelEnvVar := strings.Join(tunnels, ",")
// TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable // TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable
os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar) os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar) xlog.Debug("setting LLAMACPP_GRPC_SERVERS", "value", tunnelEnvVar)
}), }),
} }
@@ -152,17 +151,17 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
token := "" token := ""
if r.Peer2Peer || r.Peer2PeerToken != "" { if r.Peer2Peer || r.Peer2PeerToken != "" {
log.Info().Msg("P2P mode enabled") xlog.Info("P2P mode enabled")
token = r.Peer2PeerToken token = r.Peer2PeerToken
if token == "" { if token == "" {
// IF no token is provided, and p2p is enabled, // IF no token is provided, and p2p is enabled,
// we generate one and wait for the user to pick up the token (this is for interactive) // we generate one and wait for the user to pick up the token (this is for interactive)
log.Info().Msg("No token provided, generating one") xlog.Info("No token provided, generating one")
token = p2p.GenerateToken(r.Peer2PeerDHTInterval, r.Peer2PeerOTPInterval) token = p2p.GenerateToken(r.Peer2PeerDHTInterval, r.Peer2PeerOTPInterval)
log.Info().Msg("Generated Token:") xlog.Info("Generated Token:")
fmt.Println(token) fmt.Println(token)
log.Info().Msg("To use the token, you can run the following command in another node or terminal:") xlog.Info("To use the token, you can run the following command in another node or terminal:")
fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token) fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token)
} }
opts = append(opts, config.WithP2PToken(token)) opts = append(opts, config.WithP2PToken(token))
@@ -248,7 +247,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
appHTTP, err := http.API(app) appHTTP, err := http.API(app)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error during HTTP App construction") xlog.Error("error during HTTP App construction", "error", err)
return err return err
} }
@@ -260,7 +259,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
signals.RegisterGracefulTerminationHandler(func() { signals.RegisterGracefulTerminationHandler(func() {
if err := app.ModelLoader().StopAllGRPC(); err != nil { if err := app.ModelLoader().StopAllGRPC(); err != nil {
log.Error().Err(err).Msg("error while stopping all grpc backends") xlog.Error("error while stopping all grpc backends", "error", err)
} }
}) })

4
core/cli/soundgeneration.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type SoundGenerationCMD struct { type SoundGenerationCMD struct {
@@ -84,7 +84,7 @@ func (t *SoundGenerationCMD) Run(ctx *cliContext.Context) error {
defer func() { defer func() {
err := ml.StopAllGRPC() err := ml.StopAllGRPC()
if err != nil { if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes") xlog.Error("unable to stop all grpc processes", "error", err)
} }
}() }()

4
core/cli/transcript.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type TranscriptCMD struct { type TranscriptCMD struct {
@@ -54,7 +54,7 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
defer func() { defer func() {
err := ml.StopAllGRPC() err := ml.StopAllGRPC()
if err != nil { if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes") xlog.Error("unable to stop all grpc processes", "error", err)
} }
}() }()

4
core/cli/tts.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type TTSCMD struct { type TTSCMD struct {
@@ -53,7 +53,7 @@ func (t *TTSCMD) Run(ctx *cliContext.Context) error {
defer func() { defer func() {
err := ml.StopAllGRPC() err := ml.StopAllGRPC()
if err != nil { if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes") xlog.Error("unable to stop all grpc processes", "error", err)
} }
}() }()

50
core/cli/util.go
View File

@@ -9,7 +9,7 @@ import (
"strings" "strings"
"github.com/mholt/archiver/v3" "github.com/mholt/archiver/v3"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
gguf "github.com/gpustack/gguf-parser-go" gguf "github.com/gpustack/gguf-parser-go"
cliContext "github.com/mudler/LocalAI/core/cli/context" cliContext "github.com/mudler/LocalAI/core/cli/context"
@@ -51,7 +51,7 @@ type CreateOCIImageCMD struct {
} }
func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error { func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
log.Info().Msg("Creating OCI image from input") xlog.Info("Creating OCI image from input")
dir, err := os.MkdirTemp("", "localai") dir, err := os.MkdirTemp("", "localai")
if err != nil { if err != nil {
@@ -62,7 +62,7 @@ func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
if err != nil { if err != nil {
return err return err
} }
log.Info().Msgf("Creating '%s' as '%s' from %v", u.Output, u.Input, u.Input) xlog.Info("Creating OCI image", "output", u.Output, "input", u.Input)
platform := strings.Split(u.Platform, "/") platform := strings.Split(u.Platform, "/")
if len(platform) != 2 { if len(platform) != 2 {
@@ -80,27 +80,23 @@ func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error {
f, err := gguf.ParseGGUFFile(u.Args[0]) f, err := gguf.ParseGGUFFile(u.Args[0])
if err != nil { if err != nil {
// Only valid for gguf files // Only valid for gguf files
log.Error().Msgf("guessDefaultsFromFile: %s", "not a GGUF file") xlog.Error("guessDefaultsFromFile: not a GGUF file")
return err return err
} }
log.Info(). xlog.Info("GGUF file loaded", "file", u.Args[0], "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
Any("eosTokenID", f.Tokenizer().EOSTokenID).
Any("bosTokenID", f.Tokenizer().BOSTokenID).
Any("modelName", f.Metadata().Name).
Any("architecture", f.Architecture().Architecture).Msgf("GGUF file loaded: %s", u.Args[0])
log.Info().Any("tokenizer", fmt.Sprintf("%+v", f.Tokenizer())).Msg("Tokenizer") xlog.Info("Tokenizer", "tokenizer", fmt.Sprintf("%+v", f.Tokenizer()))
log.Info().Any("architecture", fmt.Sprintf("%+v", f.Architecture())).Msg("Architecture") xlog.Info("Architecture", "architecture", fmt.Sprintf("%+v", f.Architecture()))
v, exists := f.Header.MetadataKV.Get("tokenizer.chat_template") v, exists := f.Header.MetadataKV.Get("tokenizer.chat_template")
if exists { if exists {
log.Info().Msgf("chat_template: %s", v.ValueString()) xlog.Info("chat_template", "template", v.ValueString())
} }
if u.Header { if u.Header {
for _, metadata := range f.Header.MetadataKV { for _, metadata := range f.Header.MetadataKV {
log.Info().Msgf("%s: %+v", metadata.Key, metadata.Value) xlog.Info("metadata", "key", metadata.Key, "value", metadata.Value)
} }
// log.Info().Any("header", fmt.Sprintf("%+v", f.Header)).Msg("Header") // log.Info().Any("header", fmt.Sprintf("%+v", f.Header)).Msg("Header")
} }
@@ -117,63 +113,63 @@ func (hfscmd *HFScanCMD) Run(ctx *cliContext.Context) error {
return err return err
} }
log.Info().Msg("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!") xlog.Info("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!")
if len(hfscmd.ToScan) == 0 { if len(hfscmd.ToScan) == 0 {
log.Info().Msg("Checking all installed models against galleries") xlog.Info("Checking all installed models against galleries")
var galleries []config.Gallery var galleries []config.Gallery
if err := json.Unmarshal([]byte(hfscmd.Galleries), &galleries); err != nil { if err := json.Unmarshal([]byte(hfscmd.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries") xlog.Error("unable to load galleries", "error", err)
} }
err := gallery.SafetyScanGalleryModels(galleries, systemState) err := gallery.SafetyScanGalleryModels(galleries, systemState)
if err == nil { if err == nil {
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.") xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
} else { } else {
log.Error().Err(err).Msg("! WARNING ! A known-vulnerable model is installed!") xlog.Error("! WARNING ! A known-vulnerable model is installed!", "error", err)
} }
return err return err
} else { } else {
var errs error = nil var errs error = nil
for _, uri := range hfscmd.ToScan { for _, uri := range hfscmd.ToScan {
log.Info().Str("uri", uri).Msg("scanning specific uri") xlog.Info("scanning specific uri", "uri", uri)
scanResults, err := downloader.HuggingFaceScan(downloader.URI(uri)) scanResults, err := downloader.HuggingFaceScan(downloader.URI(uri))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) { if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Err(err).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("! WARNING ! A known-vulnerable model is included in this repo!") xlog.Error("! WARNING ! A known-vulnerable model is included in this repo!", "error", err, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
errs = errors.Join(errs, err) errs = errors.Join(errs, err)
} }
} }
if errs != nil { if errs != nil {
return errs return errs
} }
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.") xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
return nil return nil
} }
} }
func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error { func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error {
if len(uhcmd.ConfigName) == 0 { if len(uhcmd.ConfigName) == 0 {
log.Error().Msg("ConfigName is a required parameter") xlog.Error("ConfigName is a required parameter")
return fmt.Errorf("config name is a required parameter") return fmt.Errorf("config name is a required parameter")
} }
if len(uhcmd.ModelsPath) == 0 { if len(uhcmd.ModelsPath) == 0 {
log.Error().Msg("ModelsPath is a required parameter") xlog.Error("ModelsPath is a required parameter")
return fmt.Errorf("model path is a required parameter") return fmt.Errorf("model path is a required parameter")
} }
bcl := config.NewModelConfigLoader(uhcmd.ModelsPath) bcl := config.NewModelConfigLoader(uhcmd.ModelsPath)
err := bcl.ReadModelConfig(uhcmd.ConfigName) err := bcl.ReadModelConfig(uhcmd.ConfigName)
if err != nil { if err != nil {
log.Error().Err(err).Str("ConfigName", uhcmd.ConfigName).Msg("error while loading backend") xlog.Error("error while loading backend", "error", err, "ConfigName", uhcmd.ConfigName)
return err return err
} }
bc, exists := bcl.GetModelConfig(uhcmd.ConfigName) bc, exists := bcl.GetModelConfig(uhcmd.ConfigName)
if !exists { if !exists {
log.Error().Str("ConfigName", uhcmd.ConfigName).Msg("ConfigName not found") xlog.Error("ConfigName not found", "ConfigName", uhcmd.ConfigName)
} }
for name, uc := range config.GetAllModelConfigUsecases() { for name, uc := range config.GetAllModelConfigUsecases() {
if bc.HasUsecases(uc) { if bc.HasUsecases(uc) {
log.Info().Str("Usecase", name) xlog.Info("Usecase", "usecase", name)
} }
} }
log.Info().Msg("---") xlog.Info("---")
return nil return nil
} }

10
core/cli/worker/worker_llamacpp.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type LLamaCPP struct { type LLamaCPP struct {
@@ -30,22 +30,22 @@ const (
func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) { func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) {
backends, err := gallery.ListSystemBackends(systemState) backends, err := gallery.ListSystemBackends(systemState)
if err != nil { if err != nil {
log.Warn().Msgf("Failed listing system backends: %s", err) xlog.Warn("Failed listing system backends", "error", err)
return "", err return "", err
} }
log.Debug().Msgf("System backends: %v", backends) xlog.Debug("System backends", "backends", backends)
backend, ok := backends.Get(llamaCPPGalleryName) backend, ok := backends.Get(llamaCPPGalleryName)
if !ok { if !ok {
ml := model.NewModelLoader(systemState) ml := model.NewModelLoader(systemState)
var gals []config.Gallery var gals []config.Gallery
if err := json.Unmarshal([]byte(galleries), &gals); err != nil { if err := json.Unmarshal([]byte(galleries), &gals); err != nil {
log.Error().Err(err).Msg("failed loading galleries") xlog.Error("failed loading galleries", "error", err)
return "", err return "", err
} }
err := gallery.InstallBackendFromGallery(context.Background(), gals, systemState, ml, llamaCPPGalleryName, nil, true) err := gallery.InstallBackendFromGallery(context.Background(), gals, systemState, ml, llamaCPPGalleryName, nil, true)
if err != nil { if err != nil {
log.Error().Err(err).Msg("llama-cpp backend not found, failed to install it") xlog.Error("llama-cpp backend not found, failed to install it", "error", err)
return "", err return "", err
} }
} }

12
core/cli/worker/worker_p2p.go
View File

@@ -12,8 +12,8 @@ import (
"github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/pkg/signals" "github.com/mudler/LocalAI/pkg/signals"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
"github.com/phayes/freeport" "github.com/phayes/freeport"
"github.com/rs/zerolog/log"
) )
type P2P struct { type P2P struct {
@@ -66,16 +66,16 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
if err != nil { if err != nil {
return err return err
} }
log.Info().Msgf("You need to start llama-cpp-rpc-server on '%s:%s'", address, p) xlog.Info("You need to start llama-cpp-rpc-server", "address", address, "port", p)
} else { } else {
// Start llama.cpp directly from the version we have pre-packaged // Start llama.cpp directly from the version we have pre-packaged
go func() { go func() {
for { for {
log.Info().Msgf("Starting llama-cpp-rpc-server on '%s:%d'", address, port) xlog.Info("Starting llama-cpp-rpc-server", "address", address, "port", port)
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState) grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed to find llama-cpp-rpc-server") xlog.Error("Failed to find llama-cpp-rpc-server", "error", err)
return return
} }
@@ -85,7 +85,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ") extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ")
} }
args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...) args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...)
log.Debug().Msgf("Starting llama-cpp-rpc-server on '%s:%d' with args: %+v (%d)", address, port, args, len(args)) xlog.Debug("Starting llama-cpp-rpc-server", "address", address, "port", port, "args", args, "argCount", len(args))
cmd := exec.Command( cmd := exec.Command(
grpcProcess, args..., grpcProcess, args...,
@@ -97,7 +97,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
log.Error().Any("grpcProcess", grpcProcess).Any("args", args).Err(err).Msg("Failed to start llama-cpp-rpc-server") xlog.Error("Failed to start llama-cpp-rpc-server", "error", err, "grpcProcess", grpcProcess, "args", args)
} }
cmd.Wait() cmd.Wait()

8
core/config/application_config.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type ApplicationConfig struct { type ApplicationConfig struct {
@@ -298,7 +298,7 @@ func WithStringGalleries(galls string) AppOption {
} }
var galleries []Gallery var galleries []Gallery
if err := json.Unmarshal([]byte(galls), &galleries); err != nil { if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
log.Error().Err(err).Msg("failed loading galleries") xlog.Error("failed loading galleries", "error", err)
} }
o.Galleries = append(o.Galleries, galleries...) o.Galleries = append(o.Galleries, galleries...)
} }
@@ -312,7 +312,7 @@ func WithBackendGalleries(galls string) AppOption {
} }
var galleries []Gallery var galleries []Gallery
if err := json.Unmarshal([]byte(galls), &galleries); err != nil { if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
log.Error().Err(err).Msg("failed loading galleries") xlog.Error("failed loading galleries", "error", err)
} }
o.BackendGalleries = append(o.BackendGalleries, galleries...) o.BackendGalleries = append(o.BackendGalleries, galleries...)
} }
@@ -470,7 +470,7 @@ func WithHttpGetExemptedEndpoints(endpoints []string) AppOption {
if err == nil && r != nil { if err == nil && r != nil {
o.HttpGetExemptedEndpoints = append(o.HttpGetExemptedEndpoints, r) o.HttpGetExemptedEndpoints = append(o.HttpGetExemptedEndpoints, r)
} else { } else {
log.Warn().Err(err).Str("regex", epr).Msg("Error while compiling HTTP Get Exemption regex, skipping this entry.") xlog.Warn("Error while compiling HTTP Get Exemption regex, skipping this entry.", "error", err, "regex", epr)
} }
} }
} }

22
core/config/gguf.go
View File

@@ -2,7 +2,7 @@ package config
import ( import (
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
gguf "github.com/gpustack/gguf-parser-go" gguf "github.com/gpustack/gguf-parser-go"
) )
@@ -35,22 +35,22 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
// vram estimation // vram estimation
vram, err := xsysinfo.TotalAvailableVRAM() vram, err := xsysinfo.TotalAvailableVRAM()
if err != nil { if err != nil {
log.Error().Msgf("guessDefaultsFromFile(TotalAvailableVRAM): %s", err) xlog.Error("guessDefaultsFromFile(TotalAvailableVRAM)", "error", err)
} else if vram > 0 { } else if vram > 0 {
estimate, err := xsysinfo.EstimateGGUFVRAMUsage(f, vram) estimate, err := xsysinfo.EstimateGGUFVRAMUsage(f, vram)
if err != nil { if err != nil {
log.Error().Msgf("guessDefaultsFromFile(EstimateGGUFVRAMUsage): %s", err) xlog.Error("guessDefaultsFromFile(EstimateGGUFVRAMUsage)", "error", err)
} else { } else {
if estimate.IsFullOffload { if estimate.IsFullOffload {
log.Warn().Msgf("guessDefaultsFromFile: %s", "full offload is recommended") xlog.Warn("guessDefaultsFromFile: full offload is recommended")
} }
if estimate.EstimatedVRAM > vram { if estimate.EstimatedVRAM > vram {
log.Warn().Msgf("guessDefaultsFromFile: %s", "estimated VRAM usage is greater than available VRAM") xlog.Warn("guessDefaultsFromFile: estimated VRAM usage is greater than available VRAM")
} }
if cfg.NGPULayers == nil && estimate.EstimatedLayers > 0 { if cfg.NGPULayers == nil && estimate.EstimatedLayers > 0 {
log.Debug().Msgf("guessDefaultsFromFile: %d layers estimated", estimate.EstimatedLayers) xlog.Debug("guessDefaultsFromFile: layers estimated", "layers", estimate.EstimatedLayers)
cfg.NGPULayers = &estimate.EstimatedLayers cfg.NGPULayers = &estimate.EstimatedLayers
} }
} }
@@ -62,20 +62,16 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
cfg.NGPULayers = &defaultHigh cfg.NGPULayers = &defaultHigh
} }
log.Debug().Any("NGPULayers", cfg.NGPULayers).Msgf("guessDefaultsFromFile: %s", "NGPULayers set") xlog.Debug("guessDefaultsFromFile: NGPULayers set", "NGPULayers", cfg.NGPULayers)
// template estimations // template estimations
if cfg.HasTemplate() { if cfg.HasTemplate() {
// nothing to guess here // nothing to guess here
log.Debug().Any("name", cfg.Name).Msgf("guessDefaultsFromFile: %s", "template already set") xlog.Debug("guessDefaultsFromFile: template already set", "name", cfg.Name)
return return
} }
log.Debug(). xlog.Debug("Model file loaded", "file", cfg.ModelFileName(), "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
Any("eosTokenID", f.Tokenizer().EOSTokenID).
Any("bosTokenID", f.Tokenizer().BOSTokenID).
Any("modelName", f.Metadata().Name).
Any("architecture", f.Architecture().Architecture).Msgf("Model file loaded: %s", cfg.ModelFileName())
// guess the name // guess the name
if cfg.Name == "" { if cfg.Name == "" {

8
core/config/guesser.go
View File

@@ -5,17 +5,17 @@ import (
"path/filepath" "path/filepath"
gguf "github.com/gpustack/gguf-parser-go" gguf "github.com/gpustack/gguf-parser-go"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) { func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" { if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
log.Debug().Msgf("guessDefaultsFromFile: %s", "guessing disabled with LOCALAI_DISABLE_GUESSING") xlog.Debug("guessDefaultsFromFile: guessing disabled with LOCALAI_DISABLE_GUESSING")
return return
} }
if modelPath == "" { if modelPath == "" {
log.Debug().Msgf("guessDefaultsFromFile: %s", "modelPath is empty") xlog.Debug("guessDefaultsFromFile: modelPath is empty")
return return
} }
@@ -24,7 +24,7 @@ func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
log.Error().Msgf("guessDefaultsFromFile: %s", "panic while parsing gguf file") xlog.Error("guessDefaultsFromFile: panic while parsing gguf file")
} }
}() }()

750
core/config/model_config_loader.go
View File

@@ -1,375 +1,375 @@
package config package config
import ( import (
"errors" "errors"
"fmt" "fmt"
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"github.com/charmbracelet/glamour" "github.com/charmbracelet/glamour"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
type ModelConfigLoader struct { type ModelConfigLoader struct {
configs map[string]ModelConfig configs map[string]ModelConfig
modelPath string modelPath string
sync.Mutex sync.Mutex
} }
func NewModelConfigLoader(modelPath string) *ModelConfigLoader { func NewModelConfigLoader(modelPath string) *ModelConfigLoader {
return &ModelConfigLoader{ return &ModelConfigLoader{
configs: make(map[string]ModelConfig), configs: make(map[string]ModelConfig),
modelPath: modelPath, modelPath: modelPath,
} }
} }
type LoadOptions struct { type LoadOptions struct {
modelPath string modelPath string
debug bool debug bool
threads, ctxSize int threads, ctxSize int
f16 bool f16 bool
} }
func LoadOptionDebug(debug bool) ConfigLoaderOption { func LoadOptionDebug(debug bool) ConfigLoaderOption {
return func(o *LoadOptions) { return func(o *LoadOptions) {
o.debug = debug o.debug = debug
} }
} }
func LoadOptionThreads(threads int) ConfigLoaderOption { func LoadOptionThreads(threads int) ConfigLoaderOption {
return func(o *LoadOptions) { return func(o *LoadOptions) {
o.threads = threads o.threads = threads
} }
} }
func LoadOptionContextSize(ctxSize int) ConfigLoaderOption { func LoadOptionContextSize(ctxSize int) ConfigLoaderOption {
return func(o *LoadOptions) { return func(o *LoadOptions) {
o.ctxSize = ctxSize o.ctxSize = ctxSize
} }
} }
func ModelPath(modelPath string) ConfigLoaderOption { func ModelPath(modelPath string) ConfigLoaderOption {
return func(o *LoadOptions) { return func(o *LoadOptions) {
o.modelPath = modelPath o.modelPath = modelPath
} }
} }
func LoadOptionF16(f16 bool) ConfigLoaderOption { func LoadOptionF16(f16 bool) ConfigLoaderOption {
return func(o *LoadOptions) { return func(o *LoadOptions) {
o.f16 = f16 o.f16 = f16
} }
} }
type ConfigLoaderOption func(*LoadOptions) type ConfigLoaderOption func(*LoadOptions)
func (lo *LoadOptions) Apply(options ...ConfigLoaderOption) { func (lo *LoadOptions) Apply(options ...ConfigLoaderOption) {
for _, l := range options { for _, l := range options {
l(lo) l(lo)
} }
} }
// TODO: either in the next PR or the next commit, I want to merge these down into a single function that looks at the first few characters of the file to determine if we need to deserialize to []BackendConfig or BackendConfig // TODO: either in the next PR or the next commit, I want to merge these down into a single function that looks at the first few characters of the file to determine if we need to deserialize to []BackendConfig or BackendConfig
func readMultipleModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*ModelConfig, error) { func readMultipleModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*ModelConfig, error) {
c := &[]*ModelConfig{} c := &[]*ModelConfig{}
f, err := os.ReadFile(file) f, err := os.ReadFile(file)
if err != nil { if err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot read config file %q: %w", file, err) return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot read config file %q: %w", file, err)
} }
if err := yaml.Unmarshal(f, c); err != nil { if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot unmarshal config file %q: %w", file, err) return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot unmarshal config file %q: %w", file, err)
} }
for _, cc := range *c { for _, cc := range *c {
cc.modelConfigFile = file cc.modelConfigFile = file
cc.SetDefaults(opts...) cc.SetDefaults(opts...)
} }
return *c, nil return *c, nil
} }
func readModelConfigFromFile(file string, opts ...ConfigLoaderOption) (*ModelConfig, error) { func readModelConfigFromFile(file string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
lo := &LoadOptions{} lo := &LoadOptions{}
lo.Apply(opts...) lo.Apply(opts...)
c := &ModelConfig{} c := &ModelConfig{}
f, err := os.ReadFile(file) f, err := os.ReadFile(file)
if err != nil { if err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot read config file %q: %w", file, err) return nil, fmt.Errorf("readModelConfigFromFile cannot read config file %q: %w", file, err)
} }
if err := yaml.Unmarshal(f, c); err != nil { if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot unmarshal config file %q: %w", file, err) return nil, fmt.Errorf("readModelConfigFromFile cannot unmarshal config file %q: %w", file, err)
} }
c.SetDefaults(opts...) c.SetDefaults(opts...)
c.modelConfigFile = file c.modelConfigFile = file
return c, nil return c, nil
} }
// Load a config file for a model // Load a config file for a model
func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath string, opts ...ConfigLoaderOption) (*ModelConfig, error) { func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
// Load a config file if present after the model name // Load a config file if present after the model name
cfg := &ModelConfig{ cfg := &ModelConfig{
PredictionOptions: schema.PredictionOptions{ PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{ BasicModelRequest: schema.BasicModelRequest{
Model: modelName, Model: modelName,
}, },
}, },
} }
cfgExisting, exists := bcl.GetModelConfig(modelName) cfgExisting, exists := bcl.GetModelConfig(modelName)
if exists { if exists {
cfg = &cfgExisting cfg = &cfgExisting
} else { } else {
// Try loading a model config file // Try loading a model config file
modelConfig := filepath.Join(modelPath, modelName+".yaml") modelConfig := filepath.Join(modelPath, modelName+".yaml")
if _, err := os.Stat(modelConfig); err == nil { if _, err := os.Stat(modelConfig); err == nil {
if err := bcl.ReadModelConfig( if err := bcl.ReadModelConfig(
modelConfig, opts..., modelConfig, opts...,
); err != nil { ); err != nil {
return nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error()) return nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error())
} }
cfgExisting, exists = bcl.GetModelConfig(modelName) cfgExisting, exists = bcl.GetModelConfig(modelName)
if exists { if exists {
cfg = &cfgExisting cfg = &cfgExisting
} }
} }
} }
cfg.SetDefaults(append(opts, ModelPath(modelPath))...) cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
return cfg, nil return cfg, nil
} }
func (bcl *ModelConfigLoader) LoadModelConfigFileByNameDefaultOptions(modelName string, appConfig *ApplicationConfig) (*ModelConfig, error) { func (bcl *ModelConfigLoader) LoadModelConfigFileByNameDefaultOptions(modelName string, appConfig *ApplicationConfig) (*ModelConfig, error) {
return bcl.LoadModelConfigFileByName(modelName, appConfig.SystemState.Model.ModelsPath, return bcl.LoadModelConfigFileByName(modelName, appConfig.SystemState.Model.ModelsPath,
LoadOptionDebug(appConfig.Debug), LoadOptionDebug(appConfig.Debug),
LoadOptionThreads(appConfig.Threads), LoadOptionThreads(appConfig.Threads),
LoadOptionContextSize(appConfig.ContextSize), LoadOptionContextSize(appConfig.ContextSize),
LoadOptionF16(appConfig.F16), LoadOptionF16(appConfig.F16),
ModelPath(appConfig.SystemState.Model.ModelsPath)) ModelPath(appConfig.SystemState.Model.ModelsPath))
} }
// This format is currently only used when reading a single file at startup, passed in via ApplicationConfig.ConfigFile // This format is currently only used when reading a single file at startup, passed in via ApplicationConfig.ConfigFile
func (bcl *ModelConfigLoader) LoadMultipleModelConfigsSingleFile(file string, opts ...ConfigLoaderOption) error { func (bcl *ModelConfigLoader) LoadMultipleModelConfigsSingleFile(file string, opts ...ConfigLoaderOption) error {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
c, err := readMultipleModelConfigsFromFile(file, opts...) c, err := readMultipleModelConfigsFromFile(file, opts...)
if err != nil { if err != nil {
return fmt.Errorf("cannot load config file: %w", err) return fmt.Errorf("cannot load config file: %w", err)
} }
for _, cc := range c { for _, cc := range c {
if valid, _ := cc.Validate(); valid { if valid, _ := cc.Validate(); valid {
bcl.configs[cc.Name] = *cc bcl.configs[cc.Name] = *cc
} }
} }
return nil return nil
} }
func (bcl *ModelConfigLoader) ReadModelConfig(file string, opts ...ConfigLoaderOption) error { func (bcl *ModelConfigLoader) ReadModelConfig(file string, opts ...ConfigLoaderOption) error {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
c, err := readModelConfigFromFile(file, opts...) c, err := readModelConfigFromFile(file, opts...)
if err != nil { if err != nil {
return fmt.Errorf("ReadModelConfig cannot read config file %q: %w", file, err) return fmt.Errorf("ReadModelConfig cannot read config file %q: %w", file, err)
} }
if valid, _ := c.Validate(); valid { if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c bcl.configs[c.Name] = *c
} else { } else {
return fmt.Errorf("config is not valid") return fmt.Errorf("config is not valid")
} }
return nil return nil
} }
func (bcl *ModelConfigLoader) GetModelConfig(m string) (ModelConfig, bool) { func (bcl *ModelConfigLoader) GetModelConfig(m string) (ModelConfig, bool) {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
v, exists := bcl.configs[m] v, exists := bcl.configs[m]
return v, exists return v, exists
} }
func (bcl *ModelConfigLoader) GetAllModelsConfigs() []ModelConfig { func (bcl *ModelConfigLoader) GetAllModelsConfigs() []ModelConfig {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
var res []ModelConfig var res []ModelConfig
for _, v := range bcl.configs { for _, v := range bcl.configs {
res = append(res, v) res = append(res, v)
} }
sort.SliceStable(res, func(i, j int) bool { sort.SliceStable(res, func(i, j int) bool {
return res[i].Name < res[j].Name return res[i].Name < res[j].Name
}) })
return res return res
} }
func (bcl *ModelConfigLoader) GetModelConfigsByFilter(filter ModelConfigFilterFn) []ModelConfig { func (bcl *ModelConfigLoader) GetModelConfigsByFilter(filter ModelConfigFilterFn) []ModelConfig {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
var res []ModelConfig var res []ModelConfig
if filter == nil { if filter == nil {
filter = NoFilterFn filter = NoFilterFn
} }
for n, v := range bcl.configs { for n, v := range bcl.configs {
if filter(n, &v) { if filter(n, &v) {
res = append(res, v) res = append(res, v)
} }
} }
// TODO: I don't think this one needs to Sort on name... but we'll see what breaks. // TODO: I don't think this one needs to Sort on name... but we'll see what breaks.
return res return res
} }
func (bcl *ModelConfigLoader) RemoveModelConfig(m string) { func (bcl *ModelConfigLoader) RemoveModelConfig(m string) {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
delete(bcl.configs, m) delete(bcl.configs, m)
} }
// Preload prepare models if they are not local but url or huggingface repositories // Preload prepare models if they are not local but url or huggingface repositories
func (bcl *ModelConfigLoader) Preload(modelPath string) error { func (bcl *ModelConfigLoader) Preload(modelPath string) error {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
status := func(fileName, current, total string, percent float64) { status := func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent) utils.DisplayDownloadFunction(fileName, current, total, percent)
} }
log.Info().Msgf("Preloading models from %s", modelPath) xlog.Info("Preloading models", "path", modelPath)
renderMode := "dark" renderMode := "dark"
if os.Getenv("COLOR") != "" { if os.Getenv("COLOR") != "" {
renderMode = os.Getenv("COLOR") renderMode = os.Getenv("COLOR")
} }
glamText := func(t string) { glamText := func(t string) {
out, err := glamour.Render(t, renderMode) out, err := glamour.Render(t, renderMode)
if err == nil && os.Getenv("NO_COLOR") == "" { if err == nil && os.Getenv("NO_COLOR") == "" {
fmt.Println(out) fmt.Println(out)
} else { } else {
fmt.Println(t) fmt.Println(t)
} }
} }
for i, config := range bcl.configs { for i, config := range bcl.configs {
// Download files and verify their SHA // Download files and verify their SHA
for i, file := range config.DownloadFiles { for i, file := range config.DownloadFiles {
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename) xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, modelPath); err != nil { if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
return err return err
} }
// Create file path // Create file path
filePath := filepath.Join(modelPath, file.Filename) filePath := filepath.Join(modelPath, file.Filename)
if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil { if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil {
return err return err
} }
} }
// If the model is an URL, expand it, and download the file // If the model is an URL, expand it, and download the file
if config.IsModelURL() { if config.IsModelURL() {
modelFileName := config.ModelFileName() modelFileName := config.ModelFileName()
uri := downloader.URI(config.Model) uri := downloader.URI(config.Model)
if uri.ResolveURL() != config.Model { if uri.ResolveURL() != config.Model {
// check if file exists // check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status) err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil { if err != nil {
return err return err
} }
} }
cc := bcl.configs[i] cc := bcl.configs[i]
c := &cc c := &cc
c.PredictionOptions.Model = modelFileName c.PredictionOptions.Model = modelFileName
bcl.configs[i] = *c bcl.configs[i] = *c
} }
} }
if config.IsMMProjURL() { if config.IsMMProjURL() {
modelFileName := config.MMProjFileName() modelFileName := config.MMProjFileName()
uri := downloader.URI(config.MMProj) uri := downloader.URI(config.MMProj)
// check if file exists // check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status) err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil { if err != nil {
return err return err
} }
} }
cc := bcl.configs[i] cc := bcl.configs[i]
c := &cc c := &cc
c.MMProj = modelFileName c.MMProj = modelFileName
bcl.configs[i] = *c bcl.configs[i] = *c
} }
if bcl.configs[i].Name != "" { if bcl.configs[i].Name != "" {
glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name)) glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name))
} }
if bcl.configs[i].Description != "" { if bcl.configs[i].Description != "" {
//glamText("**Description**") //glamText("**Description**")
glamText(bcl.configs[i].Description) glamText(bcl.configs[i].Description)
} }
if bcl.configs[i].Usage != "" { if bcl.configs[i].Usage != "" {
//glamText("**Usage**") //glamText("**Usage**")
glamText(bcl.configs[i].Usage) glamText(bcl.configs[i].Usage)
} }
} }
return nil return nil
} }
// LoadModelConfigsFromPath reads all the configurations of the models from a path // LoadModelConfigsFromPath reads all the configurations of the models from a path
// (non-recursive) // (non-recursive)
func (bcl *ModelConfigLoader) LoadModelConfigsFromPath(path string, opts ...ConfigLoaderOption) error { func (bcl *ModelConfigLoader) LoadModelConfigsFromPath(path string, opts ...ConfigLoaderOption) error {
bcl.Lock() bcl.Lock()
defer bcl.Unlock() defer bcl.Unlock()
entries, err := os.ReadDir(path) entries, err := os.ReadDir(path)
if err != nil { if err != nil {
return fmt.Errorf("LoadModelConfigsFromPath cannot read directory '%s': %w", path, err) return fmt.Errorf("LoadModelConfigsFromPath cannot read directory '%s': %w", path, err)
} }
files := make([]fs.FileInfo, 0, len(entries)) files := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries { for _, entry := range entries {
info, err := entry.Info() info, err := entry.Info()
if err != nil { if err != nil {
return err return err
} }
files = append(files, info) files = append(files, info)
} }
for _, file := range files { for _, file := range files {
// Skip templates, YAML and .keep files // Skip templates, YAML and .keep files
if !strings.Contains(file.Name(), ".yaml") && !strings.Contains(file.Name(), ".yml") || if !strings.Contains(file.Name(), ".yaml") && !strings.Contains(file.Name(), ".yml") ||
strings.HasPrefix(file.Name(), ".") { strings.HasPrefix(file.Name(), ".") {
continue continue
} }
c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...) c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...)
if err != nil { if err != nil {
log.Error().Err(err).Str("File Name", file.Name()).Msgf("LoadModelConfigsFromPath cannot read config file") xlog.Error("LoadModelConfigsFromPath cannot read config file", "error", err, "File Name", file.Name())
continue continue
} }
if valid, _ := c.Validate(); valid { if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c bcl.configs[c.Name] = *c
} else { } else {
log.Error().Err(err).Str("Name", c.Name).Msgf("config is not valid") xlog.Error("config is not valid", "error", err, "Name", c.Name)
} }
} }
return nil return nil
} }

14
core/explorer/discovery.go
View File

@@ -7,7 +7,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
@@ -57,21 +57,21 @@ func (s *DiscoveryServer) runBackground() {
// do not do in parallel // do not do in parallel
n, err := p2p.NewNode(token) n, err := p2p.NewNode(token)
if err != nil { if err != nil {
log.Err(err).Msg("Failed to create node") xlog.Error("Failed to create node", "error", err)
s.failedToken(token) s.failedToken(token)
continue continue
} }
err = n.Start(c) err = n.Start(c)
if err != nil { if err != nil {
log.Err(err).Msg("Failed to start node") xlog.Error("Failed to start node", "error", err)
s.failedToken(token) s.failedToken(token)
continue continue
} }
ledger, err := n.Ledger() ledger, err := n.Ledger()
if err != nil { if err != nil {
log.Err(err).Msg("Failed to start ledger") xlog.Error("Failed to start ledger", "error", err)
s.failedToken(token) s.failedToken(token)
continue continue
} }
@@ -92,10 +92,10 @@ func (s *DiscoveryServer) runBackground() {
} }
} }
log.Debug().Any("network", token).Msgf("Network has %d clusters", len(ledgerK)) xlog.Debug("Network clusters", "network", token, "count", len(ledgerK))
if len(ledgerK) != 0 { if len(ledgerK) != 0 {
for _, k := range ledgerK { for _, k := range ledgerK {
log.Debug().Any("network", token).Msgf("Clusterdata %+v", k) xlog.Debug("Clusterdata", "network", token, "cluster", k)
} }
} }
@@ -128,7 +128,7 @@ func (s *DiscoveryServer) deleteFailedConnections() {
for _, t := range s.database.TokenList() { for _, t := range s.database.TokenList() {
data, _ := s.database.Get(t) data, _ := s.database.Get(t)
if data.Failures > s.errorThreshold { if data.Failures > s.errorThreshold {
log.Info().Any("token", t).Msg("Token has been removed from the database") xlog.Info("Token has been removed from the database", "token", t)
s.database.Delete(t) s.database.Delete(t)
} }
} }

6
core/gallery/backend_types.go
View File

@@ -5,7 +5,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// BackendMetadata represents the metadata stored in a JSON file for each installed backend // BackendMetadata represents the metadata stored in a JSON file for each installed backend
@@ -37,11 +37,11 @@ func (backend *GalleryBackend) FindBestBackendFromMeta(systemState *system.Syste
realBackend := backend.CapabilitiesMap[systemState.Capability(backend.CapabilitiesMap)] realBackend := backend.CapabilitiesMap[systemState.Capability(backend.CapabilitiesMap)]
if realBackend == "" { if realBackend == "" {
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("No backend found for reported capability") xlog.Debug("No backend found for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
return nil return nil
} }
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("Found backend for reported capability") xlog.Debug("Found backend for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
return backends.FindByName(realBackend) return backends.FindByName(realBackend)
} }

26
core/gallery/backends.go
View File

@@ -16,8 +16,8 @@ import (
"github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
cp "github.com/otiai10/copy" cp "github.com/otiai10/copy"
"github.com/rs/zerolog/log"
) )
const ( const (
@@ -86,7 +86,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
return fmt.Errorf("backend name is empty") return fmt.Errorf("backend name is empty")
} }
log.Debug().Interface("galleries", galleries).Str("name", name).Msg("Installing backend from gallery") xlog.Debug("Installing backend from gallery", "galleries", galleries, "name", name)
backends, err := AvailableBackends(galleries, systemState) backends, err := AvailableBackends(galleries, systemState)
if err != nil { if err != nil {
@@ -99,7 +99,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
} }
if backend.IsMeta() { if backend.IsMeta() {
log.Debug().Interface("systemState", systemState).Str("name", name).Msg("Backend is a meta backend") xlog.Debug("Backend is a meta backend", "systemState", systemState, "name", name)
// Then, let's try to find the best backend based on the capabilities map // Then, let's try to find the best backend based on the capabilities map
bestBackend := backend.FindBestBackendFromMeta(systemState, backends) bestBackend := backend.FindBestBackendFromMeta(systemState, backends)
@@ -107,7 +107,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
return fmt.Errorf("no backend found with capabilities %q", backend.CapabilitiesMap) return fmt.Errorf("no backend found with capabilities %q", backend.CapabilitiesMap)
} }
log.Debug().Str("name", name).Str("bestBackend", bestBackend.Name).Msg("Installing backend from meta backend") xlog.Debug("Installing backend from meta backend", "name", name, "bestBackend", bestBackend.Name)
// Then, let's install the best backend // Then, let's install the best backend
if err := InstallBackend(ctx, systemState, modelLoader, bestBackend, downloadStatus); err != nil { if err := InstallBackend(ctx, systemState, modelLoader, bestBackend, downloadStatus); err != nil {
@@ -164,7 +164,7 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
return fmt.Errorf("failed copying: %w", err) return fmt.Errorf("failed copying: %w", err)
} }
} else { } else {
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloading backend") xlog.Debug("Downloading backend", "uri", config.URI, "backendPath", backendPath)
if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil { if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil {
success := false success := false
// Try to download from mirrors // Try to download from mirrors
@@ -177,24 +177,24 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
} }
if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil { if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
success = true success = true
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend") xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
break break
} }
} }
if !success { if !success {
log.Error().Str("uri", config.URI).Str("backendPath", backendPath).Err(err).Msg("Failed to download backend") xlog.Error("Failed to download backend", "uri", config.URI, "backendPath", backendPath, "error", err)
return fmt.Errorf("failed to download backend %q: %v", config.URI, err) return fmt.Errorf("failed to download backend %q: %v", config.URI, err)
} }
} else { } else {
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend") xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
} }
} }
// sanity check - check if runfile is present // sanity check - check if runfile is present
runFile := filepath.Join(backendPath, runFile) runFile := filepath.Join(backendPath, runFile)
if _, err := os.Stat(runFile); os.IsNotExist(err) { if _, err := os.Stat(runFile); os.IsNotExist(err) {
log.Error().Str("runFile", runFile).Msg("Run file not found") xlog.Error("Run file not found", "runFile", runFile)
return fmt.Errorf("not a valid backend: run file not found %q", runFile) return fmt.Errorf("not a valid backend: run file not found %q", runFile)
} }
@@ -271,7 +271,7 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error
if metadata != nil && metadata.MetaBackendFor != "" { if metadata != nil && metadata.MetaBackendFor != "" {
metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor) metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor)
log.Debug().Str("backendDirectory", metaBackendDirectory).Msg("Deleting meta backend") xlog.Debug("Deleting meta backend", "backendDirectory", metaBackendDirectory)
if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) { if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) {
return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor) return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor)
} }
@@ -330,9 +330,9 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
} }
} }
} else if !errors.Is(err, os.ErrNotExist) { } else if !errors.Is(err, os.ErrNotExist) {
log.Warn().Err(err).Msg("Failed to read system backends, proceeding with user-managed backends") xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err)
} else if errors.Is(err, os.ErrNotExist) { } else if errors.Is(err, os.ErrNotExist) {
log.Debug().Msg("No system backends found") xlog.Debug("No system backends found")
} }
// User-managed backends and alias collection // User-managed backends and alias collection
@@ -442,7 +442,7 @@ func RegisterBackends(systemState *system.SystemState, modelLoader *model.ModelL
} }
for _, backend := range backends { for _, backend := range backends {
log.Debug().Str("name", backend.Name).Str("runFile", backend.RunFile).Msg("Registering backend") xlog.Debug("Registering backend", "name", backend.Name, "runFile", backend.RunFile)
modelLoader.SetExternalBackend(backend.Name, backend.RunFile) modelLoader.SetExternalBackend(backend.Name, backend.RunFile)
} }

8
core/gallery/gallery.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsync" "github.com/mudler/LocalAI/pkg/xsync"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
@@ -26,7 +26,7 @@ func GetGalleryConfigFromURL[T any](url string, basePath string) (T, error) {
return yaml.Unmarshal(d, &config) return yaml.Unmarshal(d, &config)
}) })
if err != nil { if err != nil {
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url") xlog.Error("failed to get gallery config for url", "error", err, "url", url)
return config, err return config, err
} }
return config, nil return config, nil
@@ -39,7 +39,7 @@ func GetGalleryConfigFromURLWithContext[T any](ctx context.Context, url string,
return yaml.Unmarshal(d, &config) return yaml.Unmarshal(d, &config)
}) })
if err != nil { if err != nil {
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url") xlog.Error("failed to get gallery config for url", "error", err, "url", url)
return config, err return config, err
} }
return config, nil return config, nil
@@ -310,7 +310,7 @@ func getGalleryElements[T GalleryElement](gallery config.Gallery, basePath strin
}) })
if err != nil { if err != nil {
if yamlErr, ok := err.(*yaml.TypeError); ok { if yamlErr, ok := err.(*yaml.TypeError); ok {
log.Debug().Msgf("YAML errors: %s\n\nwreckage of models: %+v", strings.Join(yamlErr.Errors, "\n"), models) xlog.Debug("YAML errors", "errors", strings.Join(yamlErr.Errors, "\n"), "models", models)
} }
return models, fmt.Errorf("failed to read gallery elements: %w", err) return models, fmt.Errorf("failed to read gallery elements: %w", err)
} }

12
core/gallery/importers/importers.go
View File

@@ -6,7 +6,7 @@ import (
"os" "os"
"strings" "strings"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
@@ -52,10 +52,10 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
if err != nil { if err != nil {
// maybe not a HF repository // maybe not a HF repository
// TODO: maybe we can check if the URI is a valid HF repository // TODO: maybe we can check if the URI is a valid HF repository
log.Debug().Str("uri", uri).Str("hfrepoID", hfrepoID).Msg("Failed to get model details, maybe not a HF repository") xlog.Debug("Failed to get model details, maybe not a HF repository", "uri", uri, "hfrepoID", hfrepoID)
} else { } else {
log.Debug().Str("uri", uri).Msg("Got model details") xlog.Debug("Got model details", "uri", uri)
log.Debug().Any("details", hfDetails).Msg("Model details") xlog.Debug("Model details", "details", hfDetails)
} }
// handle local config files ("/my-model.yaml" or "file://my-model.yaml") // handle local config files ("/my-model.yaml" or "file://my-model.yaml")
@@ -73,13 +73,13 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
return nil return nil
}) })
if err != nil { if err != nil {
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition") xlog.Error("error reading model definition", "error", err, "filepath", localURI)
return gallery.ModelConfig{}, err return gallery.ModelConfig{}, err
} }
} else { } else {
modelYAML, err = os.ReadFile(localURI) modelYAML, err = os.ReadFile(localURI)
if err != nil { if err != nil {
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition") xlog.Error("error reading model definition", "error", err, "filepath", localURI)
return gallery.ModelConfig{}, err return gallery.ModelConfig{}, err
} }
} }

8
core/gallery/importers/llama-cpp.go
View File

@@ -11,7 +11,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/functions"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"go.yaml.in/yaml/v2" "go.yaml.in/yaml/v2"
) )
@@ -22,7 +22,7 @@ type LlamaCPPImporter struct{}
func (i *LlamaCPPImporter) Match(details Details) bool { func (i *LlamaCPPImporter) Match(details Details) bool {
preferences, err := details.Preferences.MarshalJSON() preferences, err := details.Preferences.MarshalJSON()
if err != nil { if err != nil {
log.Error().Err(err).Msg("failed to marshal preferences") xlog.Error("failed to marshal preferences", "error", err)
return false return false
} }
@@ -31,7 +31,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
if len(preferences) > 0 { if len(preferences) > 0 {
err = json.Unmarshal(preferences, &preferencesMap) err = json.Unmarshal(preferences, &preferencesMap)
if err != nil { if err != nil {
log.Error().Err(err).Msg("failed to unmarshal preferences") xlog.Error("failed to unmarshal preferences", "error", err)
return false return false
} }
} }
@@ -63,7 +63,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) { func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) {
log.Debug().Str("uri", details.URI).Msg("llama.cpp importer matched") xlog.Debug("llama.cpp importer matched", "uri", details.URI)
preferences, err := details.Preferences.MarshalJSON() preferences, err := details.Preferences.MarshalJSON()
if err != nil { if err != nil {

34
core/gallery/models.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -131,9 +131,9 @@ func InstallModelFromGallery(
if err != nil { if err != nil {
return err return err
} }
log.Debug().Msgf("Installed model %q", installedModel.Name) xlog.Debug("Installed model", "model", installedModel.Name)
if automaticallyInstallBackend && installedModel.Backend != "" { if automaticallyInstallBackend && installedModel.Backend != "" {
log.Debug().Msgf("Installing backend %q", installedModel.Backend) xlog.Debug("Installing backend", "backend", installedModel.Backend)
if err := InstallBackendFromGallery(ctx, backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil { if err := InstallBackendFromGallery(ctx, backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil {
return err return err
@@ -165,7 +165,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
} }
if len(configOverrides) > 0 { if len(configOverrides) > 0 {
log.Debug().Msgf("Config overrides %+v", configOverrides) xlog.Debug("Config overrides", "overrides", configOverrides)
} }
// Download files and verify their SHA // Download files and verify their SHA
@@ -177,7 +177,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
default: default:
} }
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename) xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, basePath); err != nil { if err := utils.VerifyPath(file.Filename, basePath); err != nil {
return nil, err return nil, err
@@ -189,7 +189,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
if enforceScan { if enforceScan {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI)) scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) { if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", config.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!") xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return nil, err return nil, err
} }
} }
@@ -218,7 +218,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err) return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
} }
log.Debug().Msgf("Prompt template %q written", template.Name) xlog.Debug("Prompt template written", "template", template.Name)
} }
name := config.Name name := config.Name
@@ -269,7 +269,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, fmt.Errorf("failed to write updated config file: %v", err) return nil, fmt.Errorf("failed to write updated config file: %v", err)
} }
log.Debug().Msgf("Written config file %s", configFilePath) xlog.Debug("Written config file", "file", configFilePath)
} }
// Save the model gallery file for further reference // Save the model gallery file for further reference
@@ -279,7 +279,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, err return nil, err
} }
log.Debug().Msgf("Written gallery file %s", modelFile) xlog.Debug("Written gallery file", "file", modelFile)
return &modelConfig, os.WriteFile(modelFile, data, 0600) return &modelConfig, os.WriteFile(modelFile, data, 0600)
} }
@@ -341,7 +341,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
allFiles = append(allFiles, fullPath) allFiles = append(allFiles, fullPath)
} }
} else { } else {
log.Error().Err(err).Msgf("failed to read gallery file %s", configFile) xlog.Error("failed to read gallery file", "error", err, "file", configFile)
} }
for _, f := range additionalFiles { for _, f := range additionalFiles {
@@ -391,26 +391,26 @@ func DeleteModelFromSystem(systemState *system.SystemState, name string) error {
name := strings.TrimSuffix(f.Name(), ".yaml") name := strings.TrimSuffix(f.Name(), ".yaml")
name = strings.TrimSuffix(name, ".yml") name = strings.TrimSuffix(name, ".yml")
log.Debug().Msgf("Checking file %s", f.Name()) xlog.Debug("Checking file", "file", f.Name())
files, err := listModelFiles(systemState, name) files, err := listModelFiles(systemState, name)
if err != nil { if err != nil {
log.Debug().Err(err).Msgf("failed to list files for model %s", f.Name()) xlog.Debug("failed to list files for model", "error", err, "model", f.Name())
continue continue
} }
allOtherFiles = append(allOtherFiles, files...) allOtherFiles = append(allOtherFiles, files...)
} }
log.Debug().Msgf("Files to remove: %+v", filesToRemove) xlog.Debug("Files to remove", "files", filesToRemove)
log.Debug().Msgf("All other files: %+v", allOtherFiles) xlog.Debug("All other files", "files", allOtherFiles)
// Removing files // Removing files
for _, f := range filesToRemove { for _, f := range filesToRemove {
if slices.Contains(allOtherFiles, f) { if slices.Contains(allOtherFiles, f) {
log.Debug().Msgf("Skipping file %s because it is part of another model", f) xlog.Debug("Skipping file because it is part of another model", "file", f)
continue continue
} }
if e := os.Remove(f); e != nil { if e := os.Remove(f); e != nil {
log.Error().Err(e).Msgf("failed to remove file %s", f) xlog.Error("failed to remove file", "error", e, "file", f)
} }
} }
@@ -436,7 +436,7 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
for _, file := range galleryModel.AdditionalFiles { for _, file := range galleryModel.AdditionalFiles {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI)) scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) { if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", galleryModel.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!") xlog.Error("Contains unsafe file(s)!", "model", galleryModel.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return err return err
} }
} }

15
core/http/app.go
View File

@@ -21,7 +21,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// Embed a directory // Embed a directory
@@ -101,18 +101,13 @@ func API(application *application.Application) (*echo.Echo, error) {
}) })
} }
// Custom logger middleware using zerolog // Custom logger middleware using xlog
e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error { return func(c echo.Context) error {
req := c.Request() req := c.Request()
res := c.Response() res := c.Response()
start := log.Logger.Info()
err := next(c) err := next(c)
start. xlog.Info("HTTP request", "method", req.Method, "path", req.URL.Path, "status", res.Status)
Str("method", req.Method).
Str("path", req.URL.Path).
Int("status", res.Status).
Msg("HTTP request")
return err return err
} }
}) })
@@ -193,7 +188,7 @@ func API(application *application.Application) (*echo.Echo, error) {
// CSRF middleware // CSRF middleware
if application.ApplicationConfig().CSRF { if application.ApplicationConfig().CSRF {
log.Debug().Msg("Enabling CSRF middleware. Tokens are now required for state-modifying requests") xlog.Debug("Enabling CSRF middleware. Tokens are now required for state-modifying requests")
e.Use(middleware.CSRF()) e.Use(middleware.CSRF())
} }
@@ -219,7 +214,7 @@ func API(application *application.Application) (*echo.Echo, error) {
// Log startup message // Log startup message
e.Server.RegisterOnShutdown(func() { e.Server.RegisterOnShutdown(func() {
log.Info().Msg("LocalAI API server shutting down") xlog.Info("LocalAI API server shutting down")
}) })
return e, nil return e, nil

10
core/http/app_test.go
View File

@@ -25,8 +25,8 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"github.com/mudler/xlog"
openaigo "github.com/otiai10/openaigo" openaigo "github.com/otiai10/openaigo"
"github.com/rs/zerolog/log"
"github.com/sashabaranov/go-openai" "github.com/sashabaranov/go-openai"
"github.com/sashabaranov/go-openai/jsonschema" "github.com/sashabaranov/go-openai/jsonschema"
) )
@@ -378,7 +378,7 @@ var _ = Describe("API test", func() {
go func() { go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed { if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error") xlog.Error("server error", "error", err)
} }
}() }()
@@ -710,7 +710,7 @@ parameters:
go func() { go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed { if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error") xlog.Error("server error", "error", err)
} }
}() }()
@@ -924,7 +924,7 @@ parameters:
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
go func() { go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed { if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error") xlog.Error("server error", "error", err)
} }
}() }()
@@ -1405,7 +1405,7 @@ parameters:
go func() { go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed { if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error") xlog.Error("server error", "error", err)
} }
}() }()

4
core/http/endpoints/elevenlabs/soundgeneration.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// SoundGenerationEndpoint is the ElevenLabs SoundGeneration endpoint https://elevenlabs.io/docs/api-reference/sound-generation // SoundGenerationEndpoint is the ElevenLabs SoundGeneration endpoint https://elevenlabs.io/docs/api-reference/sound-generation
@@ -30,7 +30,7 @@ func SoundGenerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Sound Generation Request about to be sent to backend") xlog.Debug("Sound Generation Request about to be sent to backend", "modelFile", "modelFile", "backend", cfg.Backend)
// TODO: Support uploading files? // TODO: Support uploading files?
filePath, _, err := backend.SoundGeneration(input.Text, input.Duration, input.Temperature, input.DoSample, nil, nil, ml, appConfig, *cfg) filePath, _, err := backend.SoundGeneration(input.Text, input.Duration, input.Temperature, input.DoSample, nil, nil, ml, appConfig, *cfg)

4
core/http/endpoints/elevenlabs/tts.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech // TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
@@ -33,7 +33,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("modelName", input.ModelID).Msg("elevenlabs TTS request received") xlog.Debug("elevenlabs TTS request received", "modelName", input.ModelID)
filePath, _, err := backend.ModelTTS(input.Text, voiceID, input.LanguageCode, ml, appConfig, *cfg) filePath, _, err := backend.ModelTTS(input.Text, voiceID, input.LanguageCode, ml, appConfig, *cfg)
if err != nil { if err != nil {

4
core/http/endpoints/jina/rerank.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// JINARerankEndpoint acts like the Jina reranker endpoint (https://jina.ai/reranker/) // JINARerankEndpoint acts like the Jina reranker endpoint (https://jina.ai/reranker/)
@@ -31,7 +31,7 @@ func JINARerankEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("model", input.Model).Msg("JINA Rerank Request received") xlog.Debug("JINA Rerank Request received", "model", input.Model)
var requestTopN int32 var requestTopN int32
docs := int32(len(input.Documents)) docs := int32(len(input.Documents))
if input.TopN == nil { // omit top_n to get all if input.TopN == nil { // omit top_n to get all

6
core/http/endpoints/localai/backend.go
View File

@@ -4,15 +4,15 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/labstack/echo/v4"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type BackendEndpointService struct { type BackendEndpointService struct {
@@ -131,7 +131,7 @@ func (mgs *BackendEndpointService) ListBackendsEndpoint(systemState *system.Syst
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents! // NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc { func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc {
return func(c echo.Context) error { return func(c echo.Context) error {
log.Debug().Msgf("Listing backend galleries %+v", mgs.galleries) xlog.Debug("Listing backend galleries", "galleries", mgs.galleries)
dat, err := json.Marshal(mgs.galleries) dat, err := json.Marshal(mgs.galleries)
if err != nil { if err != nil {
return err return err

4
core/http/endpoints/localai/detection.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// DetectionEndpoint is the LocalAI Detection endpoint https://localai.io/docs/api-reference/detection // DetectionEndpoint is the LocalAI Detection endpoint https://localai.io/docs/api-reference/detection
@@ -29,7 +29,7 @@ func DetectionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appC
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("image", input.Image).Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Detection") xlog.Debug("Detection", "image", input.Image, "modelFile", "modelFile", "backend", cfg.Backend)
image, err := utils.GetContentURIAsBase64(input.Image) image, err := utils.GetContentURIAsBase64(input.Image)
if err != nil { if err != nil {

12
core/http/endpoints/localai/gallery.go
View File

@@ -4,15 +4,15 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/labstack/echo/v4"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type ModelGalleryEndpointService struct { type ModelGalleryEndpointService struct {
@@ -121,11 +121,11 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
models, err := gallery.AvailableGalleryModels(mgs.galleries, systemState) models, err := gallery.AvailableGalleryModels(mgs.galleries, systemState)
if err != nil { if err != nil {
log.Error().Err(err).Msg("could not list models from galleries") xlog.Error("could not list models from galleries", "error", err)
return err return err
} }
log.Debug().Msgf("Available %d models from %d galleries\n", len(models), len(mgs.galleries)) xlog.Debug("Available models from galleries", "modelCount", len(models), "galleryCount", len(mgs.galleries))
m := []gallery.Metadata{} m := []gallery.Metadata{}
@@ -133,7 +133,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
m = append(m, mm.Metadata) m = append(m, mm.Metadata)
} }
log.Debug().Msgf("Models %#v", m) xlog.Debug("Models", "models", m)
dat, err := json.Marshal(m) dat, err := json.Marshal(m)
if err != nil { if err != nil {
@@ -150,7 +150,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents! // NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
func (mgs *ModelGalleryEndpointService) ListModelGalleriesEndpoint() echo.HandlerFunc { func (mgs *ModelGalleryEndpointService) ListModelGalleriesEndpoint() echo.HandlerFunc {
return func(c echo.Context) error { return func(c echo.Context) error {
log.Debug().Msgf("Listing model galleries %+v", mgs.galleries) xlog.Debug("Listing model galleries", "galleries", mgs.galleries)
dat, err := json.Marshal(mgs.galleries) dat, err := json.Marshal(mgs.galleries)
if err != nil { if err != nil {
return err return err

10
core/http/endpoints/localai/get_token_metrics.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
) )
@@ -34,19 +34,19 @@ func TokenMetricsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a
modelFile, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_NAME).(string) modelFile, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if !ok || modelFile != "" { if !ok || modelFile != "" {
modelFile = input.Model modelFile = input.Model
log.Warn().Msgf("Model not found in context: %s", input.Model) xlog.Warn("Model not found in context", "model", input.Model)
} }
cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(modelFile, appConfig) cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(modelFile, appConfig)
if err != nil { if err != nil {
log.Err(err) xlog.Error("Error loading model config", "error", err)
modelFile = input.Model modelFile = input.Model
log.Warn().Msgf("Model not found in context: %s", input.Model) xlog.Warn("Model not found in context", "model", input.Model)
} else { } else {
modelFile = cfg.Model modelFile = cfg.Model
} }
log.Debug().Msgf("Token Metrics for model: %s", modelFile) xlog.Debug("Token Metrics for model", "model", modelFile)
response, err := backend.TokenMetrics(modelFile, ml, appConfig, *cfg) response, err := backend.TokenMetrics(modelFile, ml, appConfig, *cfg)
if err != nil { if err != nil {

24
core/http/endpoints/localai/mcp.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/cogito" "github.com/mudler/cogito"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// MCP SSE Event Types // MCP SSE Event Types
@@ -138,19 +138,19 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
cogitoOpts = append( cogitoOpts = append(
cogitoOpts, cogitoOpts,
cogito.WithStatusCallback(func(s string) { cogito.WithStatusCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s) xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
}), }),
cogito.WithReasoningCallback(func(s string) { cogito.WithReasoningCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s) xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
}), }),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision { cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).Msg("[model agent] Tool call") xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
return cogito.ToolCallDecision{ return cogito.ToolCallDecision{
Approved: true, Approved: true,
} }
}), }),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) { cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).Msg("[model agent] Tool call result") xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
}), }),
) )
@@ -176,7 +176,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)
@@ -279,7 +279,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
select { select {
case <-ctx.Done(): case <-ctx.Done():
// Context was cancelled (client disconnected or request cancelled) // Context was cancelled (client disconnected or request cancelled)
log.Debug().Msgf("Request context cancelled, stopping stream") xlog.Debug("Request context cancelled, stopping stream")
cancel() cancel()
break LOOP break LOOP
case event := <-events: case event := <-events:
@@ -289,13 +289,13 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
} }
eventData, err := json.Marshal(event) eventData, err := json.Marshal(event)
if err != nil { if err != nil {
log.Debug().Msgf("Failed to marshal event: %v", err) xlog.Debug("Failed to marshal event", "error", err)
continue continue
} }
log.Debug().Msgf("Sending event: %s", string(eventData)) xlog.Debug("Sending event", "event", string(eventData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(eventData)) _, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(eventData))
if err != nil { if err != nil {
log.Debug().Msgf("Sending event failed: %v", err) xlog.Debug("Sending event failed", "error", err)
cancel() cancel()
return err return err
} }
@@ -307,7 +307,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
c.Response().Flush() c.Response().Flush()
break LOOP break LOOP
} }
log.Error().Msgf("Stream ended with error: %v", err) xlog.Error("Stream ended with error", "error", err)
errorEvent := MCPErrorEvent{ errorEvent := MCPErrorEvent{
Type: "error", Type: "error",
Message: err.Error(), Message: err.Error(),
@@ -324,7 +324,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
} }
} }
log.Debug().Msgf("Stream ended") xlog.Debug("Stream ended")
return nil return nil
} }
} }

12
core/http/endpoints/localai/settings.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/p2p" "github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// GetSettingsEndpoint returns current settings with precedence (env > file > defaults) // GetSettingsEndpoint returns current settings with precedence (env > file > defaults)
@@ -118,7 +118,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
if watchdogChanged { if watchdogChanged {
if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled { if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled {
if err := app.StopWatchdog(); err != nil { if err := app.StopWatchdog(); err != nil {
log.Error().Err(err).Msg("Failed to stop watchdog") xlog.Error("Failed to stop watchdog", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{ return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false, Success: false,
Error: "Settings saved but failed to stop watchdog: " + err.Error(), Error: "Settings saved but failed to stop watchdog: " + err.Error(),
@@ -126,7 +126,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
} }
} else { } else {
if err := app.RestartWatchdog(); err != nil { if err := app.RestartWatchdog(); err != nil {
log.Error().Err(err).Msg("Failed to restart watchdog") xlog.Error("Failed to restart watchdog", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{ return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false, Success: false,
Error: "Settings saved but failed to restart watchdog: " + err.Error(), Error: "Settings saved but failed to restart watchdog: " + err.Error(),
@@ -138,7 +138,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
// Restart agent job service if retention days changed // Restart agent job service if retention days changed
if agentJobChanged { if agentJobChanged {
if err := app.RestartAgentJobService(); err != nil { if err := app.RestartAgentJobService(); err != nil {
log.Error().Err(err).Msg("Failed to restart agent job service") xlog.Error("Failed to restart agent job service", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{ return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false, Success: false,
Error: "Settings saved but failed to restart agent job service: " + err.Error(), Error: "Settings saved but failed to restart agent job service: " + err.Error(),
@@ -151,7 +151,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
if p2pChanged { if p2pChanged {
if settings.P2PToken != nil && *settings.P2PToken == "" { if settings.P2PToken != nil && *settings.P2PToken == "" {
if err := app.StopP2P(); err != nil { if err := app.StopP2P(); err != nil {
log.Error().Err(err).Msg("Failed to stop P2P") xlog.Error("Failed to stop P2P", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{ return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false, Success: false,
Error: "Settings saved but failed to stop P2P: " + err.Error(), Error: "Settings saved but failed to stop P2P: " + err.Error(),
@@ -164,7 +164,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
appConfig.P2PToken = token appConfig.P2PToken = token
} }
if err := app.RestartP2P(); err != nil { if err := app.RestartP2P(); err != nil {
log.Error().Err(err).Msg("Failed to restart P2P") xlog.Error("Failed to restart P2P", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{ return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false, Success: false,
Error: "Settings saved but failed to restart P2P: " + err.Error(), Error: "Settings saved but failed to restart P2P: " + err.Error(),

4
core/http/endpoints/localai/tts.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
) )
@@ -36,7 +36,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("model", input.Model).Msg("LocalAI TTS Request received") xlog.Debug("LocalAI TTS Request received", "model", input.Model)
if cfg.Backend == "" && input.Backend != "" { if cfg.Backend == "" && input.Backend != "" {
cfg.Backend = input.Backend cfg.Backend = input.Backend

4
core/http/endpoints/localai/vad.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// VADEndpoint is Voice-Activation-Detection endpoint // VADEndpoint is Voice-Activation-Detection endpoint
@@ -28,7 +28,7 @@ func VADEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Str("model", input.Model).Msg("LocalAI VAD Request received") xlog.Debug("LocalAI VAD Request received", "model", input.Model)
resp, err := backend.VAD(input, c.Request().Context(), ml, appConfig, *cfg) resp, err := backend.VAD(input, c.Request().Context(), ml, appConfig, *cfg)

10
core/http/endpoints/localai/video.go
View File

@@ -22,7 +22,7 @@ import (
"github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/backend"
model "github.com/mudler/LocalAI/pkg/model" model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func downloadFile(url string) (string, error) { func downloadFile(url string) (string, error) {
@@ -69,13 +69,13 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
return func(c echo.Context) error { return func(c echo.Context) error {
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.VideoRequest) input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.VideoRequest)
if !ok || input.Model == "" { if !ok || input.Model == "" {
log.Error().Msg("Video Endpoint - Invalid Input") xlog.Error("Video Endpoint - Invalid Input")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || config == nil { if !ok || config == nil {
log.Error().Msg("Video Endpoint - Invalid Config") xlog.Error("Video Endpoint - Invalid Config")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
@@ -124,7 +124,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
defer os.RemoveAll(src) defer os.RemoveAll(src)
} }
log.Debug().Msgf("Parameter Config: %+v", config) xlog.Debug("Parameter Config", "config", config)
switch config.Backend { switch config.Backend {
case "stablediffusion": case "stablediffusion":
@@ -217,7 +217,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)

14
core/http/endpoints/mcp/tools.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/signals" "github.com/mudler/LocalAI/pkg/signals"
"github.com/modelcontextprotocol/go-sdk/mcp" "github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type sessionCache struct { type sessionCache struct {
@@ -47,7 +47,7 @@ func SessionsFromMCPConfig(
// Get the list of all the tools that the Agent will be esposed to // Get the list of all the tools that the Agent will be esposed to
for _, server := range remote.Servers { for _, server := range remote.Servers {
log.Debug().Msgf("[MCP remote server] Configuration : %+v", server) xlog.Debug("[MCP remote server] Configuration", "server", server)
// Create HTTP client with custom roundtripper for bearer token injection // Create HTTP client with custom roundtripper for bearer token injection
httpClient := &http.Client{ httpClient := &http.Client{
Timeout: 360 * time.Second, Timeout: 360 * time.Second,
@@ -57,16 +57,16 @@ func SessionsFromMCPConfig(
transport := &mcp.StreamableClientTransport{Endpoint: server.URL, HTTPClient: httpClient} transport := &mcp.StreamableClientTransport{Endpoint: server.URL, HTTPClient: httpClient}
mcpSession, err := client.Connect(ctx, transport, nil) mcpSession, err := client.Connect(ctx, transport, nil)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed to connect to MCP server %s", server.URL) xlog.Error("Failed to connect to MCP server", "error", err, "url", server.URL)
continue continue
} }
log.Debug().Msgf("[MCP remote server] Connected to MCP server %s", server.URL) xlog.Debug("[MCP remote server] Connected to MCP server", "url", server.URL)
cache.cache[name] = append(cache.cache[name], mcpSession) cache.cache[name] = append(cache.cache[name], mcpSession)
allSessions = append(allSessions, mcpSession) allSessions = append(allSessions, mcpSession)
} }
for _, server := range stdio.Servers { for _, server := range stdio.Servers {
log.Debug().Msgf("[MCP stdio server] Configuration : %+v", server) xlog.Debug("[MCP stdio server] Configuration", "server", server)
command := exec.Command(server.Command, server.Args...) command := exec.Command(server.Command, server.Args...)
command.Env = os.Environ() command.Env = os.Environ()
for key, value := range server.Env { for key, value := range server.Env {
@@ -75,10 +75,10 @@ func SessionsFromMCPConfig(
transport := &mcp.CommandTransport{Command: command} transport := &mcp.CommandTransport{Command: command}
mcpSession, err := client.Connect(ctx, transport, nil) mcpSession, err := client.Connect(ctx, transport, nil)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed to start MCP server %s", command) xlog.Error("Failed to start MCP server", "error", err, "command", command)
continue continue
} }
log.Debug().Msgf("[MCP stdio server] Connected to MCP server %s", command) xlog.Debug("[MCP stdio server] Connected to MCP server", "command", command)
cache.cache[name] = append(cache.cache[name], mcpSession) cache.cache[name] = append(cache.cache[name], mcpSession)
allSessions = append(allSessions, mcpSession) allSessions = append(allSessions, mcpSession)
} }

62
core/http/endpoints/openai/chat.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create // ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create
@@ -78,7 +78,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
textContentToReturn = functions.ParseTextContent(result, config.FunctionsConfig) textContentToReturn = functions.ParseTextContent(result, config.FunctionsConfig)
result = functions.CleanupLLMResult(result, config.FunctionsConfig) result = functions.CleanupLLMResult(result, config.FunctionsConfig)
functionResults := functions.ParseFunctionCall(result, config.FunctionsConfig) functionResults := functions.ParseFunctionCall(result, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn) xlog.Debug("Text content to return", "text", textContentToReturn)
noActionToRun := len(functionResults) > 0 && functionResults[0].Name == noAction || len(functionResults) == 0 noActionToRun := len(functionResults) > 0 && functionResults[0].Name == noAction || len(functionResults) == 0
switch { switch {
@@ -94,7 +94,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
result, err := handleQuestion(config, cl, req, ml, startupOptions, functionResults, result, prompt) result, err := handleQuestion(config, cl, req, ml, startupOptions, functionResults, result, prompt)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error handling question") xlog.Error("error handling question", "error", err)
return err return err
} }
usage := schema.OpenAIUsage{ usage := schema.OpenAIUsage{
@@ -195,7 +195,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Msgf("Chat endpoint configuration read: %+v", config) xlog.Debug("Chat endpoint configuration read", "config", config)
funcs := input.Functions funcs := input.Functions
shouldUseFn := len(input.Functions) > 0 && config.ShouldUseFunctions() shouldUseFn := len(input.Functions) > 0 && config.ShouldUseFunctions()
@@ -252,7 +252,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil { if err == nil {
input.Grammar = g input.Grammar = g
} else { } else {
log.Error().Err(err).Msg("Failed generating grammar") xlog.Error("Failed generating grammar", "error", err)
} }
} }
} }
@@ -260,7 +260,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
config.Grammar = input.Grammar config.Grammar = input.Grammar
if shouldUseFn { if shouldUseFn {
log.Debug().Msgf("Response needs to process functions") xlog.Debug("Response needs to process functions")
} }
switch { switch {
@@ -294,14 +294,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil { if err == nil {
config.Grammar = g config.Grammar = g
} else { } else {
log.Error().Err(err).Msg("Failed generating grammar") xlog.Error("Failed generating grammar", "error", err)
} }
case input.JSONFunctionGrammarObject != nil: case input.JSONFunctionGrammarObject != nil:
g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarOptions()...) g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarOptions()...)
if err == nil { if err == nil {
config.Grammar = g config.Grammar = g
} else { } else {
log.Error().Err(err).Msg("Failed generating grammar") xlog.Error("Failed generating grammar", "error", err)
} }
default: default:
@@ -316,7 +316,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
// functions are not supported in stream mode (yet?) // functions are not supported in stream mode (yet?)
toStream := input.Stream toStream := input.Stream
log.Debug().Msgf("Parameters: %+v", config) xlog.Debug("Parameters", "config", config)
var predInput string var predInput string
@@ -325,16 +325,16 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if !config.TemplateConfig.UseTokenizerTemplate { if !config.TemplateConfig.UseTokenizerTemplate {
predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn) predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
log.Debug().Msgf("Prompt (after templating): %s", predInput) xlog.Debug("Prompt (after templating)", "prompt", predInput)
if config.Grammar != "" { if config.Grammar != "" {
log.Debug().Msgf("Grammar: %+v", config.Grammar) xlog.Debug("Grammar", "grammar", config.Grammar)
} }
} }
switch { switch {
case toStream: case toStream:
log.Debug().Msgf("Stream request received") xlog.Debug("Stream request received")
c.Response().Header().Set("Content-Type", "text/event-stream") c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().Header().Set("Cache-Control", "no-cache") c.Response().Header().Set("Cache-Control", "no-cache")
c.Response().Header().Set("Connection", "keep-alive") c.Response().Header().Set("Connection", "keep-alive")
@@ -359,12 +359,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
select { select {
case <-input.Context.Done(): case <-input.Context.Done():
// Context was cancelled (client disconnected or request cancelled) // Context was cancelled (client disconnected or request cancelled)
log.Debug().Msgf("Request context cancelled, stopping stream") xlog.Debug("Request context cancelled, stopping stream")
input.Cancel() input.Cancel()
break LOOP break LOOP
case ev := <-responses: case ev := <-responses:
if len(ev.Choices) == 0 { if len(ev.Choices) == 0 {
log.Debug().Msgf("No choices in the response, skipping") xlog.Debug("No choices in the response, skipping")
continue continue
} }
usage = &ev.Usage // Copy a pointer to the latest usage chunk so that the stop message can reference it usage = &ev.Usage // Copy a pointer to the latest usage chunk so that the stop message can reference it
@@ -373,14 +373,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
} }
respData, err := json.Marshal(ev) respData, err := json.Marshal(ev)
if err != nil { if err != nil {
log.Debug().Msgf("Failed to marshal response: %v", err) xlog.Debug("Failed to marshal response", "error", err)
input.Cancel() input.Cancel()
continue continue
} }
log.Debug().Msgf("Sending chunk: %s", string(respData)) xlog.Debug("Sending chunk", "chunk", string(respData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData)) _, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
if err != nil { if err != nil {
log.Debug().Msgf("Sending chunk failed: %v", err) xlog.Debug("Sending chunk failed", "error", err)
input.Cancel() input.Cancel()
return err return err
} }
@@ -389,7 +389,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil { if err == nil {
break LOOP break LOOP
} }
log.Error().Msgf("Stream ended with error: %v", err) xlog.Error("Stream ended with error", "error", err)
stopReason := FinishReasonStop stopReason := FinishReasonStop
resp := &schema.OpenAIResponse{ resp := &schema.OpenAIResponse{
@@ -407,7 +407,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
} }
respData, marshalErr := json.Marshal(resp) respData, marshalErr := json.Marshal(resp)
if marshalErr != nil { if marshalErr != nil {
log.Error().Msgf("Failed to marshal error response: %v", marshalErr) xlog.Error("Failed to marshal error response", "error", marshalErr)
// Send a simple error message as fallback // Send a simple error message as fallback
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n") fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
} else { } else {
@@ -445,7 +445,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData) fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData)
fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n") fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n")
c.Response().Flush() c.Response().Flush()
log.Debug().Msgf("Stream ended") xlog.Debug("Stream ended")
return nil return nil
// no streaming mode // no streaming mode
@@ -462,14 +462,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig) textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
s = functions.CleanupLLMResult(s, config.FunctionsConfig) s = functions.CleanupLLMResult(s, config.FunctionsConfig)
results := functions.ParseFunctionCall(s, config.FunctionsConfig) results := functions.ParseFunctionCall(s, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn) xlog.Debug("Text content to return", "text", textContentToReturn)
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0 noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
switch { switch {
case noActionsToRun: case noActionsToRun:
result, err := handleQuestion(config, cl, input, ml, startupOptions, results, s, predInput) result, err := handleQuestion(config, cl, input, ml, startupOptions, results, s, predInput)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error handling question") xlog.Error("error handling question", "error", err)
return return
} }
@@ -562,7 +562,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
Usage: usage, Usage: usage,
} }
respData, _ := json.Marshal(resp) respData, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", respData) xlog.Debug("Response", "response", string(respData))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)
@@ -573,12 +573,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, input *schema.OpenAIRequest, ml *model.ModelLoader, o *config.ApplicationConfig, funcResults []functions.FuncCallResults, result, prompt string) (string, error) { func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, input *schema.OpenAIRequest, ml *model.ModelLoader, o *config.ApplicationConfig, funcResults []functions.FuncCallResults, result, prompt string) (string, error) {
if len(funcResults) == 0 && result != "" { if len(funcResults) == 0 && result != "" {
log.Debug().Msgf("nothing function results but we had a message from the LLM") xlog.Debug("nothing function results but we had a message from the LLM")
return result, nil return result, nil
} }
log.Debug().Msgf("nothing to do, computing a reply") xlog.Debug("nothing to do, computing a reply")
arg := "" arg := ""
if len(funcResults) > 0 { if len(funcResults) > 0 {
arg = funcResults[0].Arguments arg = funcResults[0].Arguments
@@ -586,23 +586,23 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
// If there is a message that the LLM already sends as part of the JSON reply, use it // If there is a message that the LLM already sends as part of the JSON reply, use it
arguments := map[string]interface{}{} arguments := map[string]interface{}{}
if err := json.Unmarshal([]byte(arg), &arguments); err != nil { if err := json.Unmarshal([]byte(arg), &arguments); err != nil {
log.Debug().Msg("handleQuestion: function result did not contain a valid JSON object") xlog.Debug("handleQuestion: function result did not contain a valid JSON object")
} }
m, exists := arguments["message"] m, exists := arguments["message"]
if exists { if exists {
switch message := m.(type) { switch message := m.(type) {
case string: case string:
if message != "" { if message != "" {
log.Debug().Msgf("Reply received from LLM: %s", message) xlog.Debug("Reply received from LLM", "message", message)
message = backend.Finetune(*config, prompt, message) message = backend.Finetune(*config, prompt, message)
log.Debug().Msgf("Reply received from LLM(finetuned): %s", message) xlog.Debug("Reply received from LLM(finetuned)", "message", message)
return message, nil return message, nil
} }
} }
} }
log.Debug().Msgf("No action received from LLM, without a message, computing a reply") xlog.Debug("No action received from LLM, without a message, computing a reply")
// Otherwise ask the LLM to understand the JSON output and the context, and return a message // Otherwise ask the LLM to understand the JSON output and the context, and return a message
// Note: This costs (in term of CPU/GPU) another computation // Note: This costs (in term of CPU/GPU) another computation
config.Grammar = "" config.Grammar = ""
@@ -662,13 +662,13 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
predFunc, err := backend.ModelInference(input.Context, prompt, input.Messages, images, videos, audios, ml, config, cl, o, nil, toolsJSON, toolChoiceJSON, logprobs, topLogprobs, logitBias) predFunc, err := backend.ModelInference(input.Context, prompt, input.Messages, images, videos, audios, ml, config, cl, o, nil, toolsJSON, toolChoiceJSON, logprobs, topLogprobs, logitBias)
if err != nil { if err != nil {
log.Error().Err(err).Msg("model inference failed") xlog.Error("model inference failed", "error", err)
return "", err return "", err
} }
prediction, err := predFunc() prediction, err := predFunc()
if err != nil { if err != nil {
log.Error().Err(err).Msg("prediction failed") xlog.Error("prediction failed", "error", err)
return "", err return "", err
} }
return backend.Finetune(*config, prompt, prediction.Response), nil return backend.Finetune(*config, prompt, prediction.Response), nil

24
core/http/endpoints/openai/completion.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions // CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
@@ -52,7 +52,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
Object: "text_completion", Object: "text_completion",
Usage: usage, Usage: usage,
} }
log.Debug().Msgf("Sending goroutine: %s", s) xlog.Debug("Sending goroutine", "text", s)
responses <- resp responses <- resp
return true return true
@@ -94,10 +94,10 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
config.Grammar = input.Grammar config.Grammar = input.Grammar
log.Debug().Msgf("Parameter Config: %+v", config) xlog.Debug("Parameter Config", "config", config)
if input.Stream { if input.Stream {
log.Debug().Msgf("Stream request received") xlog.Debug("Stream request received")
c.Response().Header().Set("Content-Type", "text/event-stream") c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().Header().Set("Cache-Control", "no-cache") c.Response().Header().Set("Cache-Control", "no-cache")
c.Response().Header().Set("Connection", "keep-alive") c.Response().Header().Set("Connection", "keep-alive")
@@ -116,7 +116,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
}) })
if err == nil { if err == nil {
predInput = templatedInput predInput = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", predInput) xlog.Debug("Template found, input modified", "input", predInput)
} }
responses := make(chan schema.OpenAIResponse) responses := make(chan schema.OpenAIResponse)
@@ -131,16 +131,16 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
select { select {
case ev := <-responses: case ev := <-responses:
if len(ev.Choices) == 0 { if len(ev.Choices) == 0 {
log.Debug().Msgf("No choices in the response, skipping") xlog.Debug("No choices in the response, skipping")
continue continue
} }
respData, err := json.Marshal(ev) respData, err := json.Marshal(ev)
if err != nil { if err != nil {
log.Debug().Msgf("Failed to marshal response: %v", err) xlog.Debug("Failed to marshal response", "error", err)
continue continue
} }
log.Debug().Msgf("Sending chunk: %s", string(respData)) xlog.Debug("Sending chunk", "chunk", string(respData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData)) _, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
if err != nil { if err != nil {
return err return err
@@ -150,7 +150,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
if err == nil { if err == nil {
break LOOP break LOOP
} }
log.Error().Msgf("Stream ended with error: %v", err) xlog.Error("Stream ended with error", "error", err)
stopReason := FinishReasonStop stopReason := FinishReasonStop
errorResp := schema.OpenAIResponse{ errorResp := schema.OpenAIResponse{
@@ -168,7 +168,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
} }
errorData, marshalErr := json.Marshal(errorResp) errorData, marshalErr := json.Marshal(errorResp)
if marshalErr != nil { if marshalErr != nil {
log.Error().Msgf("Failed to marshal error response: %v", marshalErr) xlog.Error("Failed to marshal error response", "error", marshalErr)
// Send a simple error message as fallback // Send a simple error message as fallback
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n") fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
} else { } else {
@@ -213,7 +213,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
}) })
if err == nil { if err == nil {
i = templatedInput i = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", i) xlog.Debug("Template found, input modified", "input", i)
} }
r, tokenUsage, err := ComputeChoices( r, tokenUsage, err := ComputeChoices(
@@ -250,7 +250,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)

10
core/http/endpoints/openai/edit.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// EditEndpoint is the OpenAI edit API endpoint // EditEndpoint is the OpenAI edit API endpoint
@@ -39,8 +39,8 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Msgf("Edit Endpoint Input : %+v", input) xlog.Debug("Edit Endpoint Input", "input", input)
log.Debug().Msgf("Edit Endpoint Config: %+v", *config) xlog.Debug("Edit Endpoint Config", "config", *config)
var result []schema.Choice var result []schema.Choice
totalTokenUsage := backend.TokenUsage{} totalTokenUsage := backend.TokenUsage{}
@@ -55,7 +55,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
}) })
if err == nil { if err == nil {
i = templatedInput i = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", i) xlog.Debug("Template found, input modified", "input", i)
} }
r, tokenUsage, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) { r, tokenUsage, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
@@ -95,7 +95,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)

6
core/http/endpoints/openai/embeddings.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings // EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings
@@ -33,7 +33,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return echo.ErrBadRequest return echo.ErrBadRequest
} }
log.Debug().Msgf("Parameter Config: %+v", config) xlog.Debug("Parameter Config", "config", config)
items := []schema.Item{} items := []schema.Item{}
for i, s := range config.InputToken { for i, s := range config.InputToken {
@@ -75,7 +75,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)

22
core/http/endpoints/openai/image.go
View File

@@ -23,7 +23,7 @@ import (
"github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/backend"
model "github.com/mudler/LocalAI/pkg/model" model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func downloadFile(url string) (string, error) { func downloadFile(url string) (string, error) {
@@ -70,13 +70,13 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
return func(c echo.Context) error { return func(c echo.Context) error {
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest) input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
if !ok || input.Model == "" { if !ok || input.Model == "" {
log.Error().Msg("Image Endpoint - Invalid Input") xlog.Error("Image Endpoint - Invalid Input")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || config == nil { if !ok || config == nil {
log.Error().Msg("Image Endpoint - Invalid Config") xlog.Error("Image Endpoint - Invalid Config")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
@@ -113,7 +113,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
} }
} }
log.Debug().Msgf("Parameter Config: %+v", config) xlog.Debug("Parameter Config", "config", config)
switch config.Backend { switch config.Backend {
case "stablediffusion": case "stablediffusion":
@@ -124,7 +124,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
if !strings.Contains(input.Size, "x") { if !strings.Contains(input.Size, "x") {
input.Size = "512x512" input.Size = "512x512"
log.Warn().Msgf("Invalid size, using default 512x512") xlog.Warn("Invalid size, using default 512x512")
} }
sizeParts := strings.Split(input.Size, "x") sizeParts := strings.Split(input.Size, "x")
@@ -235,7 +235,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)
@@ -251,21 +251,21 @@ func processImageFile(file string, generatedContentDir string) string {
if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") { if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") {
out, err := downloadFile(file) out, err := downloadFile(file)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed downloading file: %s", file) xlog.Error("Failed downloading file", "error", err, "file", file)
return "" return ""
} }
defer os.RemoveAll(out) defer os.RemoveAll(out)
fileData, err = os.ReadFile(out) fileData, err = os.ReadFile(out)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed reading downloaded file: %s", out) xlog.Error("Failed reading downloaded file", "error", err, "file", out)
return "" return ""
} }
} else { } else {
// base 64 decode the file and write it somewhere that we will cleanup // base 64 decode the file and write it somewhere that we will cleanup
fileData, err = base64.StdEncoding.DecodeString(file) fileData, err = base64.StdEncoding.DecodeString(file)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed decoding base64 file") xlog.Error("Failed decoding base64 file", "error", err)
return "" return ""
} }
} }
@@ -273,7 +273,7 @@ func processImageFile(file string, generatedContentDir string) string {
// Create a temporary file // Create a temporary file
outputFile, err := os.CreateTemp(generatedContentDir, "b64") outputFile, err := os.CreateTemp(generatedContentDir, "b64")
if err != nil { if err != nil {
log.Error().Err(err).Msg("Failed creating temporary file") xlog.Error("Failed creating temporary file", "error", err)
return "" return ""
} }
@@ -282,7 +282,7 @@ func processImageFile(file string, generatedContentDir string) string {
_, err = writer.Write(fileData) _, err = writer.Write(fileData)
if err != nil { if err != nil {
outputFile.Close() outputFile.Close()
log.Error().Err(err).Msg("Failed writing to temporary file") xlog.Error("Failed writing to temporary file", "error", err)
return "" return ""
} }
outputFile.Close() outputFile.Close()

34
core/http/endpoints/openai/inpainting.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/backend" "github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
@@ -48,7 +48,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
stepsStr := c.FormValue("steps") stepsStr := c.FormValue("steps")
if modelName == "" || prompt == "" { if modelName == "" || prompt == "" {
log.Error().Msg("Inpainting Endpoint - missing model or prompt") xlog.Error("Inpainting Endpoint - missing model or prompt")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
@@ -63,12 +63,12 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// Get uploaded files // Get uploaded files
imageFile, err := c.FormFile("image") imageFile, err := c.FormFile("image")
if err != nil { if err != nil {
log.Error().Err(err).Msg("Inpainting Endpoint - missing image file") xlog.Error("Inpainting Endpoint - missing image file", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "missing image file") return echo.NewHTTPError(http.StatusBadRequest, "missing image file")
} }
maskFile, err := c.FormFile("mask") maskFile, err := c.FormFile("mask")
if err != nil { if err != nil {
log.Error().Err(err).Msg("Inpainting Endpoint - missing mask file") xlog.Error("Inpainting Endpoint - missing mask file", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "missing mask file") return echo.NewHTTPError(http.StatusBadRequest, "missing mask file")
} }
@@ -100,7 +100,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// get model config from context (middleware set it) // get model config from context (middleware set it)
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || cfg == nil { if !ok || cfg == nil {
log.Error().Msg("Inpainting Endpoint - model config not found in context") xlog.Error("Inpainting Endpoint - model config not found in context")
return echo.ErrBadRequest return echo.ErrBadRequest
} }
@@ -109,7 +109,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
tmpDir := appConfig.GeneratedContentDir tmpDir := appConfig.GeneratedContentDir
// Ensure the directory exists // Ensure the directory exists
if err := os.MkdirAll(tmpDir, 0750); err != nil { if err := os.MkdirAll(tmpDir, 0750); err != nil {
log.Error().Err(err).Msgf("Inpainting Endpoint - failed to create generated content dir: %s", tmpDir) xlog.Error("Inpainting Endpoint - failed to create generated content dir", "error", err, "dir", tmpDir)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to prepare storage") return echo.NewHTTPError(http.StatusInternalServerError, "failed to prepare storage")
} }
id := uuid.New().String() id := uuid.New().String()
@@ -132,32 +132,32 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// Best-effort cleanup; log any failures // Best-effort cleanup; log any failures
if jf != nil { if jf != nil {
if cerr := jf.Close(); cerr != nil { if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file in cleanup") xlog.Warn("Inpainting Endpoint - failed to close temp json file in cleanup", "error", cerr)
} }
if name := jf.Name(); name != "" { if name := jf.Name(); name != "" {
if rerr := os.Remove(name); rerr != nil && !os.IsNotExist(rerr) { if rerr := os.Remove(name); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove temp json file %s in cleanup", name) xlog.Warn("Inpainting Endpoint - failed to remove temp json file in cleanup", "error", rerr, "file", name)
} }
} }
} }
if jsonPath != "" { if jsonPath != "" {
if rerr := os.Remove(jsonPath); rerr != nil && !os.IsNotExist(rerr) { if rerr := os.Remove(jsonPath); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove json file %s in cleanup", jsonPath) xlog.Warn("Inpainting Endpoint - failed to remove json file in cleanup", "error", rerr, "file", jsonPath)
} }
} }
if dst != "" { if dst != "" {
if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) { if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove dst file %s in cleanup", dst) xlog.Warn("Inpainting Endpoint - failed to remove dst file in cleanup", "error", rerr, "file", dst)
} }
} }
if origRef != "" { if origRef != "" {
if rerr := os.Remove(origRef); rerr != nil && !os.IsNotExist(rerr) { if rerr := os.Remove(origRef); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove orig ref file %s in cleanup", origRef) xlog.Warn("Inpainting Endpoint - failed to remove orig ref file in cleanup", "error", rerr, "file", origRef)
} }
} }
if maskRef != "" { if maskRef != "" {
if rerr := os.Remove(maskRef); rerr != nil && !os.IsNotExist(rerr) { if rerr := os.Remove(maskRef); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove mask ref file %s in cleanup", maskRef) xlog.Warn("Inpainting Endpoint - failed to remove mask ref file in cleanup", "error", rerr, "file", maskRef)
} }
} }
} }
@@ -175,7 +175,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err return err
} }
if cerr := origTmp.Close(); cerr != nil { if cerr := origTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close orig temp file") xlog.Warn("Inpainting Endpoint - failed to close orig temp file", "error", cerr)
} }
origRef = origTmp.Name() origRef = origTmp.Name()
@@ -192,19 +192,19 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err return err
} }
if cerr := maskTmp.Close(); cerr != nil { if cerr := maskTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close mask temp file") xlog.Warn("Inpainting Endpoint - failed to close mask temp file", "error", cerr)
} }
maskRef = maskTmp.Name() maskRef = maskTmp.Name()
// write JSON // write JSON
enc := json.NewEncoder(jf) enc := json.NewEncoder(jf)
if err := enc.Encode(jsonFile); err != nil { if err := enc.Encode(jsonFile); err != nil {
if cerr := jf.Close(); cerr != nil { if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file after encode error") xlog.Warn("Inpainting Endpoint - failed to close temp json file after encode error", "error", cerr)
} }
return err return err
} }
if cerr := jf.Close(); cerr != nil { if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file") xlog.Warn("Inpainting Endpoint - failed to close temp json file", "error", cerr)
} }
// rename to desired name // rename to desired name
if err := os.Rename(jf.Name(), jsonPath); err != nil { if err := os.Rename(jf.Name(), jsonPath); err != nil {
@@ -216,7 +216,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err return err
} }
if cerr := outTmp.Close(); cerr != nil { if cerr := outTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close out temp file") xlog.Warn("Inpainting Endpoint - failed to close out temp file", "error", cerr)
} }
dst = outTmp.Name() + ".png" dst = outTmp.Name() + ".png"
if err := os.Rename(outTmp.Name(), dst); err != nil { if err := os.Rename(outTmp.Name(), dst); err != nil {

12
core/http/endpoints/openai/mcp.go
View File

@@ -18,7 +18,7 @@ import (
"github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/cogito" "github.com/mudler/cogito"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// MCPCompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions // MCPCompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
@@ -102,19 +102,19 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
cogito.WithContext(ctxWithCancellation), cogito.WithContext(ctxWithCancellation),
cogito.WithMCPs(sessions...), cogito.WithMCPs(sessions...),
cogito.WithStatusCallback(func(s string) { cogito.WithStatusCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s) xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
}), }),
cogito.WithReasoningCallback(func(s string) { cogito.WithReasoningCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s) xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
}), }),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision { cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Msgf("[model agent] [model: %s] Tool call: %s, reasoning: %s, arguments: %+v", config.Name, t.Name, t.Reasoning, t.Arguments) xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
return cogito.ToolCallDecision{ return cogito.ToolCallDecision{
Approved: true, Approved: true,
} }
}), }),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) { cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Msgf("[model agent] [model: %s] Tool call result: %s, result: %s, tool arguments: %+v", config.Name, t.Name, t.Result, t.ToolArguments) xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
}), }),
) )
@@ -140,7 +140,7 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
} }
jsonResult, _ := json.Marshal(resp) jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult) xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(200, resp) return c.JSON(200, resp)

70
core/http/endpoints/openai/realtime.go
View File

@@ -27,7 +27,7 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -212,12 +212,12 @@ func registerRealtime(application *application.Application, model, intent string
return func(c *websocket.Conn) { return func(c *websocket.Conn) {
evaluator := application.TemplatesEvaluator() evaluator := application.TemplatesEvaluator()
log.Debug().Msgf("WebSocket connection established with '%s'", c.RemoteAddr().String()) xlog.Debug("WebSocket connection established", "address", c.RemoteAddr().String())
if intent != "transcription" { if intent != "transcription" {
sendNotImplemented(c, "Only transcription mode is supported which requires the intent=transcription parameter") sendNotImplemented(c, "Only transcription mode is supported which requires the intent=transcription parameter")
} }
log.Debug().Msgf("Realtime params: model=%s, intent=%s", model, intent) xlog.Debug("Realtime params", "model", model, "intent", intent)
sessionID := generateSessionID() sessionID := generateSessionID()
session := &Session{ session := &Session{
@@ -265,7 +265,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ApplicationConfig(), application.ApplicationConfig(),
) )
if err != nil { if err != nil {
log.Error().Msgf("failed to load model: %s", err.Error()) xlog.Error("failed to load model", "error", err)
sendError(c, "model_load_error", "Failed to load model", "", "") sendError(c, "model_load_error", "Failed to load model", "", "")
return return
} }
@@ -301,14 +301,14 @@ func registerRealtime(application *application.Application, model, intent string
for { for {
if _, msg, err = c.ReadMessage(); err != nil { if _, msg, err = c.ReadMessage(); err != nil {
log.Error().Msgf("read: %s", err.Error()) xlog.Error("read error", "error", err)
break break
} }
// Parse the incoming message // Parse the incoming message
var incomingMsg IncomingMessage var incomingMsg IncomingMessage
if err := json.Unmarshal(msg, &incomingMsg); err != nil { if err := json.Unmarshal(msg, &incomingMsg); err != nil {
log.Error().Msgf("invalid json: %s", err.Error()) xlog.Error("invalid json", "error", err)
sendError(c, "invalid_json", "Invalid JSON format", "", "") sendError(c, "invalid_json", "Invalid JSON format", "", "")
continue continue
} }
@@ -316,10 +316,10 @@ func registerRealtime(application *application.Application, model, intent string
var sessionUpdate types.ClientSession var sessionUpdate types.ClientSession
switch incomingMsg.Type { switch incomingMsg.Type {
case types.ClientEventTypeTranscriptionSessionUpdate: case types.ClientEventTypeTranscriptionSessionUpdate:
log.Debug().Msgf("recv: %s", msg) xlog.Debug("recv", "message", string(msg))
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil { if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
log.Error().Msgf("failed to unmarshal 'transcription_session.update': %s", err.Error()) xlog.Error("failed to unmarshal 'transcription_session.update'", "error", err)
sendError(c, "invalid_session_update", "Invalid session update format", "", "") sendError(c, "invalid_session_update", "Invalid session update format", "", "")
continue continue
} }
@@ -330,7 +330,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ModelLoader(), application.ModelLoader(),
application.ApplicationConfig(), application.ApplicationConfig(),
); err != nil { ); err != nil {
log.Error().Msgf("failed to update session: %s", err.Error()) xlog.Error("failed to update session", "error", err)
sendError(c, "session_update_error", "Failed to update session", "", "") sendError(c, "session_update_error", "Failed to update session", "", "")
continue continue
} }
@@ -344,11 +344,11 @@ func registerRealtime(application *application.Application, model, intent string
}) })
case types.ClientEventTypeSessionUpdate: case types.ClientEventTypeSessionUpdate:
log.Debug().Msgf("recv: %s", msg) xlog.Debug("recv", "message", string(msg))
// Update session configurations // Update session configurations
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil { if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
log.Error().Msgf("failed to unmarshal 'session.update': %s", err.Error()) xlog.Error("failed to unmarshal 'session.update'", "error", err)
sendError(c, "invalid_session_update", "Invalid session update format", "", "") sendError(c, "invalid_session_update", "Invalid session update format", "", "")
continue continue
} }
@@ -359,7 +359,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ModelLoader(), application.ModelLoader(),
application.ApplicationConfig(), application.ApplicationConfig(),
); err != nil { ); err != nil {
log.Error().Msgf("failed to update session: %s", err.Error()) xlog.Error("failed to update session", "error", err)
sendError(c, "session_update_error", "Failed to update session", "", "") sendError(c, "session_update_error", "Failed to update session", "", "")
continue continue
} }
@@ -373,7 +373,7 @@ func registerRealtime(application *application.Application, model, intent string
}) })
if session.TurnDetection.Type == types.ServerTurnDetectionTypeServerVad && !vadServerStarted { if session.TurnDetection.Type == types.ServerTurnDetectionTypeServerVad && !vadServerStarted {
log.Debug().Msg("Starting VAD goroutine...") xlog.Debug("Starting VAD goroutine...")
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
@@ -382,7 +382,7 @@ func registerRealtime(application *application.Application, model, intent string
}() }()
vadServerStarted = true vadServerStarted = true
} else if session.TurnDetection.Type != types.ServerTurnDetectionTypeServerVad && vadServerStarted { } else if session.TurnDetection.Type != types.ServerTurnDetectionTypeServerVad && vadServerStarted {
log.Debug().Msg("Stopping VAD goroutine...") xlog.Debug("Stopping VAD goroutine...")
wg.Add(-1) wg.Add(-1)
go func() { go func() {
@@ -393,7 +393,7 @@ func registerRealtime(application *application.Application, model, intent string
case types.ClientEventTypeInputAudioBufferAppend: case types.ClientEventTypeInputAudioBufferAppend:
// Handle 'input_audio_buffer.append' // Handle 'input_audio_buffer.append'
if incomingMsg.Audio == "" { if incomingMsg.Audio == "" {
log.Error().Msg("Audio data is missing in 'input_audio_buffer.append'") xlog.Error("Audio data is missing in 'input_audio_buffer.append'")
sendError(c, "missing_audio_data", "Audio data is missing", "", "") sendError(c, "missing_audio_data", "Audio data is missing", "", "")
continue continue
} }
@@ -401,7 +401,7 @@ func registerRealtime(application *application.Application, model, intent string
// Decode base64 audio data // Decode base64 audio data
decodedAudio, err := base64.StdEncoding.DecodeString(incomingMsg.Audio) decodedAudio, err := base64.StdEncoding.DecodeString(incomingMsg.Audio)
if err != nil { if err != nil {
log.Error().Msgf("failed to decode audio data: %s", err.Error()) xlog.Error("failed to decode audio data", "error", err)
sendError(c, "invalid_audio_data", "Failed to decode audio data", "", "") sendError(c, "invalid_audio_data", "Failed to decode audio data", "", "")
continue continue
} }
@@ -412,7 +412,7 @@ func registerRealtime(application *application.Application, model, intent string
session.AudioBufferLock.Unlock() session.AudioBufferLock.Unlock()
case types.ClientEventTypeInputAudioBufferCommit: case types.ClientEventTypeInputAudioBufferCommit:
log.Debug().Msgf("recv: %s", msg) xlog.Debug("recv", "message", string(msg))
// TODO: Trigger transcription. // TODO: Trigger transcription.
// TODO: Ignore this if VAD enabled or interrupt VAD? // TODO: Ignore this if VAD enabled or interrupt VAD?
@@ -458,12 +458,12 @@ func registerRealtime(application *application.Application, model, intent string
}) })
case types.ClientEventTypeConversationItemCreate: case types.ClientEventTypeConversationItemCreate:
log.Debug().Msgf("recv: %s", msg) xlog.Debug("recv", "message", string(msg))
// Handle creating new conversation items // Handle creating new conversation items
var item types.ConversationItemCreateEvent var item types.ConversationItemCreateEvent
if err := json.Unmarshal(incomingMsg.Item, &item); err != nil { if err := json.Unmarshal(incomingMsg.Item, &item); err != nil {
log.Error().Msgf("failed to unmarshal 'conversation.item.create': %s", err.Error()) xlog.Error("failed to unmarshal 'conversation.item.create'", "error", err)
sendError(c, "invalid_item", "Invalid item format", "", "") sendError(c, "invalid_item", "Invalid item format", "", "")
continue continue
} }
@@ -494,7 +494,7 @@ func registerRealtime(application *application.Application, model, intent string
var responseCreate types.ResponseCreateEvent var responseCreate types.ResponseCreateEvent
if len(incomingMsg.Response) > 0 { if len(incomingMsg.Response) > 0 {
if err := json.Unmarshal(incomingMsg.Response, &responseCreate); err != nil { if err := json.Unmarshal(incomingMsg.Response, &responseCreate); err != nil {
log.Error().Msgf("failed to unmarshal 'response.create' response object: %s", err.Error()) xlog.Error("failed to unmarshal 'response.create' response object", "error", err)
sendError(c, "invalid_response_create", "Invalid response create format", "", "") sendError(c, "invalid_response_create", "Invalid response create format", "", "")
continue continue
} }
@@ -515,14 +515,14 @@ func registerRealtime(application *application.Application, model, intent string
// }() // }()
case types.ClientEventTypeResponseCancel: case types.ClientEventTypeResponseCancel:
log.Printf("recv: %s", msg) xlog.Debug("recv", "message", string(msg))
// Handle cancellation of ongoing responses // Handle cancellation of ongoing responses
// Implement cancellation logic as needed // Implement cancellation logic as needed
sendNotImplemented(c, "response.cancel") sendNotImplemented(c, "response.cancel")
default: default:
log.Error().Msgf("unknown message type: %s", incomingMsg.Type) xlog.Error("unknown message type", "type", incomingMsg.Type)
sendError(c, "unknown_message_type", fmt.Sprintf("Unknown message type: %s", incomingMsg.Type), "", "") sendError(c, "unknown_message_type", fmt.Sprintf("Unknown message type: %s", incomingMsg.Type), "", "")
} }
} }
@@ -542,11 +542,11 @@ func registerRealtime(application *application.Application, model, intent string
func sendEvent(c *websocket.Conn, event types.ServerEvent) { func sendEvent(c *websocket.Conn, event types.ServerEvent) {
eventBytes, err := json.Marshal(event) eventBytes, err := json.Marshal(event)
if err != nil { if err != nil {
log.Error().Msgf("failed to marshal event: %s", err.Error()) xlog.Error("failed to marshal event", "error", err)
return return
} }
if err = c.WriteMessage(websocket.TextMessage, eventBytes); err != nil { if err = c.WriteMessage(websocket.TextMessage, eventBytes); err != nil {
log.Error().Msgf("write: %s", err.Error()) xlog.Error("write error", "error", err)
} }
} }
@@ -681,10 +681,10 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
segments, err := runVAD(vadContext, session, aints) segments, err := runVAD(vadContext, session, aints)
if err != nil { if err != nil {
if err.Error() == "unexpected speech end" { if err.Error() == "unexpected speech end" {
log.Debug().Msg("VAD cancelled") xlog.Debug("VAD cancelled")
continue continue
} }
log.Error().Msgf("failed to process audio: %s", err.Error()) xlog.Error("failed to process audio", "error", err)
sendError(c, "processing_error", "Failed to process audio: "+err.Error(), "", "") sendError(c, "processing_error", "Failed to process audio: "+err.Error(), "", "")
continue continue
} }
@@ -697,7 +697,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
session.AudioBufferLock.Lock() session.AudioBufferLock.Lock()
session.InputAudioBuffer = nil session.InputAudioBuffer = nil
session.AudioBufferLock.Unlock() session.AudioBufferLock.Unlock()
log.Debug().Msgf("Detected silence for a while, clearing audio buffer") xlog.Debug("Detected silence for a while, clearing audio buffer")
sendEvent(c, types.InputAudioBufferClearedEvent{ sendEvent(c, types.InputAudioBufferClearedEvent{
ServerEventBase: types.ServerEventBase{ ServerEventBase: types.ServerEventBase{
@@ -729,7 +729,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
} }
if float32(audioLength)-segEndTime > float32(silenceThreshold) { if float32(audioLength)-segEndTime > float32(silenceThreshold) {
log.Debug().Msgf("Detected end of speech segment") xlog.Debug("Detected end of speech segment")
session.AudioBufferLock.Lock() session.AudioBufferLock.Lock()
session.InputAudioBuffer = nil session.InputAudioBuffer = nil
session.AudioBufferLock.Unlock() session.AudioBufferLock.Unlock()
@@ -769,21 +769,21 @@ func commitUtterance(ctx context.Context, utt []byte, cfg *config.ModelConfig, e
f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav") f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav")
if err != nil { if err != nil {
log.Error().Msgf("failed to create temp file: %s", err.Error()) xlog.Error("failed to create temp file", "error", err)
return return
} }
defer f.Close() defer f.Close()
defer os.Remove(f.Name()) defer os.Remove(f.Name())
log.Debug().Msgf("Writing to %s\n", f.Name()) xlog.Debug("Writing to file", "file", f.Name())
hdr := laudio.NewWAVHeader(uint32(len(utt))) hdr := laudio.NewWAVHeader(uint32(len(utt)))
if err := hdr.Write(f); err != nil { if err := hdr.Write(f); err != nil {
log.Error().Msgf("Failed to write WAV header: %s", err.Error()) xlog.Error("Failed to write WAV header", "error", err)
return return
} }
if _, err := f.Write(utt); err != nil { if _, err := f.Write(utt); err != nil {
log.Error().Msgf("Failed to write audio data: %s", err.Error()) xlog.Error("Failed to write audio data", "error", err)
return return
} }
@@ -1106,14 +1106,14 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig) textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
s = functions.CleanupLLMResult(s, config.FunctionsConfig) s = functions.CleanupLLMResult(s, config.FunctionsConfig)
results := functions.ParseFunctionCall(s, config.FunctionsConfig) results := functions.ParseFunctionCall(s, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn) xlog.Debug("Text content to return", "text", textContentToReturn)
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0 noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
switch { switch {
case noActionsToRun: case noActionsToRun:
result, err := handleQuestion(config, input, ml, startupOptions, results, s, predInput) result, err := handleQuestion(config, input, ml, startupOptions, results, s, predInput)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error handling question") xlog.Error("error handling question", "error", err)
return return
} }
*c = append(*c, schema.Choice{ *c = append(*c, schema.Choice{
@@ -1187,7 +1187,7 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
}, },
} }
respData, _ := json.Marshal(resp) respData, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", respData) xlog.Debug("Response", "response", string(respData))
// Return the prediction in the response body // Return the prediction in the response body
return c.JSON(resp) return c.JSON(resp)

4
core/http/endpoints/openai/realtime_model.go
View File

@@ -9,7 +9,7 @@ import (
grpcClient "github.com/mudler/LocalAI/pkg/grpc" grpcClient "github.com/mudler/LocalAI/pkg/grpc"
"github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/grpc/proto"
model "github.com/mudler/LocalAI/pkg/model" model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
@@ -209,7 +209,7 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model
}, nil }, nil
} }
log.Debug().Msg("Loading a wrapped model") xlog.Debug("Loading a wrapped model")
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations // Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
cfgLLM, err := cl.LoadModelConfigFileByName(pipeline.LLM, ml.ModelPath) cfgLLM, err := cl.LoadModelConfigFileByName(pipeline.LLM, ml.ModelPath)

8
core/http/endpoints/openai/transcription.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
model "github.com/mudler/LocalAI/pkg/model" model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create // TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create
@@ -64,18 +64,18 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
} }
if _, err := io.Copy(dstFile, f); err != nil { if _, err := io.Copy(dstFile, f); err != nil {
log.Debug().Msgf("Audio file copying error %+v - %+v - err %+v", file.Filename, dst, err) xlog.Debug("Audio file copying error", "filename", file.Filename, "dst", dst, "error", err)
return err return err
} }
log.Debug().Msgf("Audio file copied to: %+v", dst) xlog.Debug("Audio file copied", "dst", dst)
tr, err := backend.ModelTranscription(dst, input.Language, input.Translate, diarize, prompt, ml, *config, appConfig) tr, err := backend.ModelTranscription(dst, input.Language, input.Translate, diarize, prompt, ml, *config, appConfig)
if err != nil { if err != nil {
return err return err
} }
log.Debug().Msgf("Trascribed: %+v", tr) xlog.Debug("Transcribed", "transcription", tr)
// TODO: handle different outputs here // TODO: handle different outputs here
return c.JSON(http.StatusOK, tr) return c.JSON(http.StatusOK, tr)
} }

4
core/http/explorer.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/explorer" "github.com/mudler/LocalAI/core/explorer"
"github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/http/routes" "github.com/mudler/LocalAI/core/http/routes"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func Explorer(db *explorer.Database) *echo.Echo { func Explorer(db *explorer.Database) *echo.Echo {
@@ -37,7 +37,7 @@ func Explorer(db *explorer.Database) *echo.Echo {
staticFS, err := fs.Sub(embedDirStatic, "static") staticFS, err := fs.Sub(embedDirStatic, "static")
if err != nil { if err != nil {
// Log error but continue - static files might not work // Log error but continue - static files might not work
log.Error().Err(err).Msg("failed to create static filesystem") xlog.Error("failed to create static filesystem", "error", err)
} else { } else {
e.StaticFS("/static", staticFS) e.StaticFS("/static", staticFS)
} }

29
core/http/middleware/request.go
View File

@@ -17,7 +17,7 @@ import (
"github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type correlationIDKeyType string type correlationIDKeyType string
@@ -82,7 +82,7 @@ func (re *RequestExtractor) BuildConstantDefaultModelNameMiddleware(defaultModel
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string) localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if !ok || localModelName == "" { if !ok || localModelName == "" {
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, defaultModelName) c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, defaultModelName)
log.Debug().Str("defaultModelName", defaultModelName).Msg("context local model name not found, setting to default") xlog.Debug("context local model name not found, setting to default", "defaultModelName", defaultModelName)
} }
return next(c) return next(c)
} }
@@ -100,19 +100,19 @@ func (re *RequestExtractor) BuildFilteredFirstAvailableDefaultModel(filterFn con
modelNames, err := services.ListModels(re.modelConfigLoader, re.modelLoader, filterFn, services.SKIP_IF_CONFIGURED) modelNames, err := services.ListModels(re.modelConfigLoader, re.modelLoader, filterFn, services.SKIP_IF_CONFIGURED)
if err != nil { if err != nil {
log.Error().Err(err).Msg("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()") xlog.Error("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()", "error", err)
return next(c) return next(c)
} }
if len(modelNames) == 0 { if len(modelNames) == 0 {
log.Warn().Msg("SetDefaultModelNameToFirstAvailable used with no matching models installed") xlog.Warn("SetDefaultModelNameToFirstAvailable used with no matching models installed")
// This is non-fatal - making it so was breaking the case of direct installation of raw models // This is non-fatal - making it so was breaking the case of direct installation of raw models
// return errors.New("this endpoint requires at least one model to be installed") // return errors.New("this endpoint requires at least one model to be installed")
return next(c) return next(c)
} }
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, modelNames[0]) c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, modelNames[0])
log.Debug().Str("first model name", modelNames[0]).Msg("context local model name not found, setting to the first model") xlog.Debug("context local model name not found, setting to the first model", "first model name", modelNames[0])
return next(c) return next(c)
} }
} }
@@ -135,7 +135,7 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
if input.ModelName(nil) == "" { if input.ModelName(nil) == "" {
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string) localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if ok && localModelName != "" { if ok && localModelName != "" {
log.Debug().Str("context localModelName", localModelName).Msg("overriding empty model name in request body with value found earlier in middleware chain") xlog.Debug("overriding empty model name in request body with value found earlier in middleware chain", "context localModelName", localModelName)
input.ModelName(&localModelName) input.ModelName(&localModelName)
} }
} }
@@ -143,10 +143,9 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig) cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig)
if err != nil { if err != nil {
log.Err(err) xlog.Warn("Model Configuration File not found", "model", input.ModelName(nil), "error", err)
log.Warn().Msgf("Model Configuration File not found for %q", input.ModelName(nil))
} else if cfg.Model == "" && input.ModelName(nil) != "" { } else if cfg.Model == "" && input.ModelName(nil) != "" {
log.Debug().Str("input.ModelName", input.ModelName(nil)).Msg("config does not include model, using input") xlog.Debug("config does not include model, using input", "input.ModelName", input.ModelName(nil))
cfg.Model = input.ModelName(nil) cfg.Model = input.ModelName(nil)
} }
@@ -203,7 +202,7 @@ func (re *RequestExtractor) SetOpenAIRequest(c echo.Context) error {
} }
if cfg.Model == "" { if cfg.Model == "" {
log.Debug().Str("input.Model", input.Model).Msg("replacing empty cfg.Model with input value") xlog.Debug("replacing empty cfg.Model with input value", "input.Model", input.Model)
cfg.Model = input.Model cfg.Model = input.Model
} }
@@ -331,7 +330,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text // Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL) base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL)
if err != nil { if err != nil {
log.Error().Msgf("Failed encoding video: %s", err) xlog.Error("Failed encoding video", "error", err)
continue CONTENT continue CONTENT
} }
input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff
@@ -341,7 +340,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text // Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL) base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL)
if err != nil { if err != nil {
log.Error().Msgf("Failed encoding audio: %s", err) xlog.Error("Failed encoding audio", "error", err)
continue CONTENT continue CONTENT
} }
input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff
@@ -356,7 +355,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text // Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL) base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL)
if err != nil { if err != nil {
log.Error().Msgf("Failed encoding image: %s", err) xlog.Error("Failed encoding image", "error", err)
continue CONTENT continue CONTENT
} }
@@ -410,7 +409,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
config.TypicalP = input.TypicalP config.TypicalP = input.TypicalP
} }
log.Debug().Str("input.Input", fmt.Sprintf("%+v", input.Input)) xlog.Debug("input.Input", "input", fmt.Sprintf("%+v", input.Input))
switch inputs := input.Input.(type) { switch inputs := input.Input.(type) {
case string: case string:
@@ -434,7 +433,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
case string: case string:
inputStrings = append(inputStrings, ii) inputStrings = append(inputStrings, ii)
default: default:
log.Error().Msgf("Unknown input type: %T", ii) xlog.Error("Unknown input type", "type", fmt.Sprintf("%T", ii))
} }
} }
config.InputToken = append(config.InputToken, tokens) config.InputToken = append(config.InputToken, tokens)

30
core/http/routes/ui_api.go
View File

@@ -20,7 +20,7 @@ import (
"github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -143,11 +143,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Cancel operation endpoint // Cancel operation endpoint
app.POST("/api/operations/:jobID/cancel", func(c echo.Context) error { app.POST("/api/operations/:jobID/cancel", func(c echo.Context) error {
jobID := c.Param("jobID") jobID := c.Param("jobID")
log.Debug().Msgf("API request to cancel operation: %s", jobID) xlog.Debug("API request to cancel operation", "jobID", jobID)
err := galleryService.CancelOperation(jobID) err := galleryService.CancelOperation(jobID)
if err != nil { if err != nil {
log.Error().Err(err).Msgf("Failed to cancel operation: %s", jobID) xlog.Error("Failed to cancel operation", "error", err, "jobID", jobID)
return c.JSON(http.StatusBadRequest, map[string]interface{}{ return c.JSON(http.StatusBadRequest, map[string]interface{}{
"error": err.Error(), "error": err.Error(),
}) })
@@ -176,7 +176,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState) models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
if err != nil { if err != nil {
log.Error().Err(err).Msg("could not list models from galleries") xlog.Error("could not list models from galleries", "error", err)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{ return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(), "error": err.Error(),
}) })
@@ -246,7 +246,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Skip duplicate IDs to prevent Alpine.js x-for errors // Skip duplicate IDs to prevent Alpine.js x-for errors
if seenIDs[modelID] { if seenIDs[modelID] {
log.Debug().Msgf("Skipping duplicate model ID: %s", modelID) xlog.Debug("Skipping duplicate model ID", "modelID", modelID)
continue continue
} }
seenIDs[modelID] = true seenIDs[modelID] = true
@@ -320,7 +320,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID", "error": "invalid model ID",
}) })
} }
log.Debug().Msgf("API job submitted to install: %+v\n", galleryID) xlog.Debug("API job submitted to install", "galleryID", galleryID)
id, err := uuid.NewUUID() id, err := uuid.NewUUID()
if err != nil { if err != nil {
@@ -362,7 +362,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID", "error": "invalid model ID",
}) })
} }
log.Debug().Msgf("API job submitted to delete: %+v\n", galleryID) xlog.Debug("API job submitted to delete", "galleryID", galleryID)
var galleryName = galleryID var galleryName = galleryID
if strings.Contains(galleryID, "@") { if strings.Contains(galleryID, "@") {
@@ -412,7 +412,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID", "error": "invalid model ID",
}) })
} }
log.Debug().Msgf("API job submitted to get config for: %+v\n", galleryID) xlog.Debug("API job submitted to get config", "galleryID", galleryID)
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState) models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
if err != nil { if err != nil {
@@ -498,7 +498,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState) backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
if err != nil { if err != nil {
log.Error().Err(err).Msg("could not list backends from galleries") xlog.Error("could not list backends from galleries", "error", err)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{ return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(), "error": err.Error(),
}) })
@@ -568,7 +568,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Skip duplicate IDs to prevent Alpine.js x-for errors // Skip duplicate IDs to prevent Alpine.js x-for errors
if seenBackendIDs[backendID] { if seenBackendIDs[backendID] {
log.Debug().Msgf("Skipping duplicate backend ID: %s", backendID) xlog.Debug("Skipping duplicate backend ID", "backendID", backendID)
continue continue
} }
seenBackendIDs[backendID] = true seenBackendIDs[backendID] = true
@@ -640,7 +640,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend ID", "error": "invalid backend ID",
}) })
} }
log.Debug().Msgf("API job submitted to install backend: %+v\n", backendID) xlog.Debug("API job submitted to install backend", "backendID", backendID)
id, err := uuid.NewUUID() id, err := uuid.NewUUID()
if err != nil { if err != nil {
@@ -695,7 +695,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
}) })
} }
log.Debug().Str("uri", req.URI).Str("name", req.Name).Str("alias", req.Alias).Msg("API job submitted to install external backend") xlog.Debug("API job submitted to install external backend", "uri", req.URI, "name", req.Name, "alias", req.Alias)
id, err := uuid.NewUUID() id, err := uuid.NewUUID()
if err != nil { if err != nil {
@@ -745,7 +745,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend ID", "error": "invalid backend ID",
}) })
} }
log.Debug().Msgf("API job submitted to delete backend: %+v\n", backendID) xlog.Debug("API job submitted to delete backend", "backendID", backendID)
var backendName = backendID var backendName = backendID
if strings.Contains(backendID, "@") { if strings.Contains(backendID, "@") {
@@ -831,11 +831,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend name", "error": "invalid backend name",
}) })
} }
log.Debug().Msgf("API request to delete system backend: %+v\n", backendName) xlog.Debug("API request to delete system backend", "backendName", backendName)
// Use the gallery package to delete the backend // Use the gallery package to delete the backend
if err := gallery.DeleteBackendFromSystem(appConfig.SystemState, backendName); err != nil { if err := gallery.DeleteBackendFromSystem(appConfig.SystemState, backendName); err != nil {
log.Error().Err(err).Msgf("Failed to delete backend: %s", backendName) xlog.Error("Failed to delete backend", "error", err, "backendName", backendName)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{ return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(), "error": err.Error(),
}) })

12
core/p2p/federated.go
View File

@@ -5,7 +5,7 @@ import (
"math/rand/v2" "math/rand/v2"
"sync" "sync"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const FederatedID = "federated" const FederatedID = "federated"
@@ -43,7 +43,7 @@ func (fs *FederatedServer) RandomServer() string {
tunnelAddresses = append(tunnelAddresses, v.ID) tunnelAddresses = append(tunnelAddresses, v.ID)
} else { } else {
delete(fs.requestTable, v.ID) // make sure it's not tracked delete(fs.requestTable, v.ID) // make sure it's not tracked
log.Info().Msgf("Node %s is offline", v.ID) xlog.Info("Node is offline", "node", v.ID)
} }
} }
@@ -80,7 +80,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
fs.Lock() fs.Lock()
defer fs.Unlock() defer fs.Unlock()
log.Debug().Any("request_table", fs.requestTable).Msgf("SelectLeastUsedServer()") xlog.Debug("SelectLeastUsedServer()", "request_table", fs.requestTable)
// cycle over requestTable and find the entry with the lower number // cycle over requestTable and find the entry with the lower number
// if there are multiple entries with the same number, select one randomly // if there are multiple entries with the same number, select one randomly
@@ -93,7 +93,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
minKey = k minKey = k
} }
} }
log.Debug().Any("requests_served", min).Any("request_table", fs.requestTable).Msgf("Selected tunnel %s", minKey) xlog.Debug("Selected tunnel", "tunnel", minKey, "requests_served", min, "request_table", fs.requestTable)
return minKey return minKey
} }
@@ -104,7 +104,7 @@ func (fs *FederatedServer) RecordRequest(nodeID string) {
// increment the counter for the nodeID in the requestTable // increment the counter for the nodeID in the requestTable
fs.requestTable[nodeID]++ fs.requestTable[nodeID]++
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Recording request") xlog.Debug("Recording request", "request_table", fs.requestTable, "request", nodeID)
} }
func (fs *FederatedServer) ensureRecordExist(nodeID string) { func (fs *FederatedServer) ensureRecordExist(nodeID string) {
@@ -114,5 +114,5 @@ func (fs *FederatedServer) ensureRecordExist(nodeID string) {
fs.requestTable[nodeID] = 0 fs.requestTable[nodeID] = 0
} }
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Ensure record exists") xlog.Debug("Ensure record exists", "request_table", fs.requestTable, "request", nodeID)
} }

22
core/p2p/federated_server.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/edgevpn/pkg/node" "github.com/mudler/edgevpn/pkg/node"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func (f *FederatedServer) Start(ctx context.Context) error { func (f *FederatedServer) Start(ctx context.Context) error {
@@ -23,7 +23,7 @@ func (f *FederatedServer) Start(ctx context.Context) error {
} }
if err := ServiceDiscoverer(ctx, n, f.p2ptoken, f.service, func(servicesID string, tunnel schema.NodeData) { if err := ServiceDiscoverer(ctx, n, f.p2ptoken, f.service, func(servicesID string, tunnel schema.NodeData) {
log.Debug().Msgf("Discovered node: %s", tunnel.ID) xlog.Debug("Discovered node", "node", tunnel.ID)
}, false); err != nil { }, false); err != nil {
return err return err
} }
@@ -33,11 +33,11 @@ func (f *FederatedServer) Start(ctx context.Context) error {
func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error { func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
log.Info().Msgf("Allocating service '%s' on: %s", fs.service, fs.listenAddr) xlog.Info("Allocating service", "service", fs.service, "address", fs.listenAddr)
// Open local port for listening // Open local port for listening
l, err := net.Listen("tcp", fs.listenAddr) l, err := net.Listen("tcp", fs.listenAddr)
if err != nil { if err != nil {
log.Error().Err(err).Msg("Error listening") xlog.Error("Error listening", "error", err)
return err return err
} }
@@ -54,7 +54,7 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
case <-ctx.Done(): case <-ctx.Done():
return errors.New("context canceled") return errors.New("context canceled")
default: default:
log.Debug().Msgf("New connection from %s", l.Addr().String()) xlog.Debug("New connection", "address", l.Addr().String())
// Listen for an incoming connection. // Listen for an incoming connection.
conn, err := l.Accept() conn, err := l.Accept()
if err != nil { if err != nil {
@@ -68,11 +68,11 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
if fs.workerTarget != "" { if fs.workerTarget != "" {
workerID = fs.workerTarget workerID = fs.workerTarget
} else if fs.loadBalanced { } else if fs.loadBalanced {
log.Debug().Msgf("Load balancing request") xlog.Debug("Load balancing request")
workerID = fs.SelectLeastUsedServer() workerID = fs.SelectLeastUsedServer()
if workerID == "" { if workerID == "" {
log.Debug().Msgf("Least used server not found, selecting random") xlog.Debug("Least used server not found, selecting random")
workerID = fs.RandomServer() workerID = fs.RandomServer()
} }
} else { } else {
@@ -80,15 +80,15 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
} }
if workerID == "" { if workerID == "" {
log.Error().Msg("No available nodes yet") xlog.Error("No available nodes yet")
fs.sendHTMLResponse(conn, 503, "Sorry, waiting for nodes to connect") fs.sendHTMLResponse(conn, 503, "Sorry, waiting for nodes to connect")
return return
} }
log.Debug().Msgf("Selected node %s", workerID) xlog.Debug("Selected node", "node", workerID)
nodeData, exists := GetNode(fs.service, workerID) nodeData, exists := GetNode(fs.service, workerID)
if !exists { if !exists {
log.Error().Msgf("Node %s not found", workerID) xlog.Error("Node not found", "node", workerID)
fs.sendHTMLResponse(conn, 404, "Node not found") fs.sendHTMLResponse(conn, 404, "Node not found")
return return
} }
@@ -123,7 +123,7 @@ func (fs *FederatedServer) sendHTMLResponse(conn net.Conn, statusCode int, messa
// Write the response to the client connection. // Write the response to the client connection.
_, writeErr := io.WriteString(conn, response) _, writeErr := io.WriteString(conn, response)
if writeErr != nil { if writeErr != nil {
log.Error().Err(writeErr).Msg("Error writing response to client") xlog.Error("Error writing response to client", "error", writeErr)
} }
} }

30
core/p2p/p2p.go
View File

@@ -21,9 +21,9 @@ import (
"github.com/mudler/edgevpn/pkg/services" "github.com/mudler/edgevpn/pkg/services"
"github.com/mudler/edgevpn/pkg/types" "github.com/mudler/edgevpn/pkg/types"
eutils "github.com/mudler/edgevpn/pkg/utils" eutils "github.com/mudler/edgevpn/pkg/utils"
zlog "github.com/mudler/xlog"
"github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multiaddr"
"github.com/phayes/freeport" "github.com/phayes/freeport"
zlog "github.com/rs/zerolog/log"
"github.com/mudler/edgevpn/pkg/logger" "github.com/mudler/edgevpn/pkg/logger"
) )
@@ -94,7 +94,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
existingValue.Unmarshal(service) existingValue.Unmarshal(service)
// If mismatch, update the blockchain // If mismatch, update the blockchain
if !found { if !found {
zlog.Error().Msg("Service not found on blockchain") zlog.Error("Service not found on blockchain")
conn.Close() conn.Close()
// ll.Debugf("service '%s' not found on blockchain", serviceID) // ll.Debugf("service '%s' not found on blockchain", serviceID)
return return
@@ -103,7 +103,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
// Decode the Peer // Decode the Peer
d, err := peer.Decode(service.PeerID) d, err := peer.Decode(service.PeerID)
if err != nil { if err != nil {
zlog.Error().Msg("cannot decode peer") zlog.Error("cannot decode peer")
conn.Close() conn.Close()
// ll.Debugf("could not decode peer '%s'", service.PeerID) // ll.Debugf("could not decode peer '%s'", service.PeerID)
@@ -113,14 +113,14 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
// Open a stream // Open a stream
stream, err := node.Host().NewStream(ctx, d, protocol.ServiceProtocol.ID()) stream, err := node.Host().NewStream(ctx, d, protocol.ServiceProtocol.ID())
if err != nil { if err != nil {
zlog.Error().Err(err).Msg("cannot open stream peer") zlog.Error("cannot open stream peer", "error", err)
conn.Close() conn.Close()
// ll.Debugf("could not open stream '%s'", err.Error()) // ll.Debugf("could not open stream '%s'", err.Error())
return return
} }
// ll.Debugf("(service %s) Redirecting", serviceID, l.Addr().String()) // ll.Debugf("(service %s) Redirecting", serviceID, l.Addr().String())
zlog.Info().Msgf("Redirecting %s to %s", conn.LocalAddr().String(), stream.Conn().RemoteMultiaddr().String()) zlog.Info("Redirecting", "from", conn.LocalAddr().String(), "to", stream.Conn().RemoteMultiaddr().String())
closer := make(chan struct{}, 2) closer := make(chan struct{}, 2)
go copyStream(closer, stream, conn) go copyStream(closer, stream, conn)
go copyStream(closer, conn, stream) go copyStream(closer, conn, stream)
@@ -131,11 +131,11 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
} }
func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error { func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error {
zlog.Info().Msgf("Allocating service '%s' on: %s", service, listenAddr) zlog.Info("Allocating service", "service", service, "address", listenAddr)
// Open local port for listening // Open local port for listening
l, err := net.Listen("tcp", listenAddr) l, err := net.Listen("tcp", listenAddr)
if err != nil { if err != nil {
zlog.Error().Err(err).Msg("Error listening") zlog.Error("Error listening", "error", err)
return err return err
} }
go func() { go func() {
@@ -151,7 +151,7 @@ func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, serv
case <-ctx.Done(): case <-ctx.Done():
return errors.New("context canceled") return errors.New("context canceled")
default: default:
zlog.Debug().Msg("New for connection") zlog.Debug("New for connection")
// Listen for an incoming connection. // Listen for an incoming connection.
conn, err := l.Accept() conn, err := l.Accept()
if err != nil { if err != nil {
@@ -187,7 +187,7 @@ func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID stri
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
zlog.Error().Msg("Discoverer stopped") zlog.Error("Discoverer stopped")
return return
case tunnel := <-tunnels: case tunnel := <-tunnels:
AddNode(servicesID, tunnel) AddNode(servicesID, tunnel)
@@ -220,7 +220,7 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
zlog.Error().Msg("Discoverer stopped") zlog.Error("Discoverer stopped")
return return
default: default:
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
@@ -230,14 +230,14 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
if logLevel == logLevelDebug { if logLevel == logLevelDebug {
// We want to surface this debugging data only if p2p logging is set to debug // We want to surface this debugging data only if p2p logging is set to debug
// (and not generally the whole application, as this can be really noisy) // (and not generally the whole application, as this can be really noisy)
zlog.Debug().Any("data", ledger.LastBlock().Storage).Msg("Ledger data") zlog.Debug("Ledger data", "data", ledger.LastBlock().Storage)
} }
for k, v := range data { for k, v := range data {
// New worker found in the ledger data as k (worker id) // New worker found in the ledger data as k (worker id)
nd := &schema.NodeData{} nd := &schema.NodeData{}
if err := v.Unmarshal(nd); err != nil { if err := v.Unmarshal(nd); err != nil {
zlog.Error().Msg("cannot unmarshal node data") zlog.Error("cannot unmarshal node data")
continue continue
} }
ensureService(ctx, n, nd, k, allocate) ensureService(ctx, n, nd, k, allocate)
@@ -278,7 +278,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
// Start the service // Start the service
port, err := freeport.GetFreePort() port, err := freeport.GetFreePort()
if err != nil { if err != nil {
zlog.Error().Err(err).Msgf("Could not allocate a free port for %s", nd.ID) zlog.Error("Could not allocate a free port", "error", err, "node", nd.ID)
cancel() cancel()
return return
} }
@@ -286,7 +286,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port) tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
nd.TunnelAddress = tunnelAddress nd.TunnelAddress = tunnelAddress
go allocateLocalService(newCtxm, n, tunnelAddress, sserv) go allocateLocalService(newCtxm, n, tunnelAddress, sserv)
zlog.Debug().Msgf("Starting service %s on %s", sserv, tunnelAddress) zlog.Debug("Starting service", "service", sserv, "address", tunnelAddress)
} }
service[nd.Name] = nodeServiceData{ service[nd.Name] = nodeServiceData{
NodeData: *nd, NodeData: *nd,
@@ -298,7 +298,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
if !nd.IsOnline() && !ndService.NodeData.IsOnline() { if !nd.IsOnline() && !ndService.NodeData.IsOnline() {
ndService.CancelFunc() ndService.CancelFunc()
delete(service, nd.Name) delete(service, nd.Name)
zlog.Info().Msgf("Node %s is offline, deleting", nd.ID) zlog.Info("Node is offline, deleting", "node", nd.ID)
} else if nd.IsOnline() { } else if nd.IsOnline() {
// update last seen inside service // update last seen inside service
nd.TunnelAddress = ndService.NodeData.TunnelAddress nd.TunnelAddress = ndService.NodeData.TunnelAddress

4
core/schema/message.go
View File

@@ -3,7 +3,7 @@ package schema
import ( import (
"encoding/json" "encoding/json"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/grpc/proto"
) )
@@ -72,7 +72,7 @@ func (messages Messages) ToProto() []*proto.Message {
if len(message.ToolCalls) > 0 { if len(message.ToolCalls) > 0 {
toolCallsJSON, err := json.Marshal(message.ToolCalls) toolCallsJSON, err := json.Marshal(message.ToolCalls)
if err != nil { if err != nil {
log.Warn().Err(err).Msg("failed to marshal tool_calls to JSON") xlog.Warn("failed to marshal tool_calls to JSON", "error", err)
} else { } else {
protoMessages[i].ToolCalls = string(toolCallsJSON) protoMessages[i].ToolCalls = string(toolCallsJSON)
} }

94
core/services/agent_jobs.go
View File

@@ -28,7 +28,7 @@ import (
"github.com/mudler/LocalAI/pkg/xsync" "github.com/mudler/LocalAI/pkg/xsync"
"github.com/mudler/cogito" "github.com/mudler/cogito"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// AgentJobService manages agent tasks and job execution // AgentJobService manages agent tasks and job execution
@@ -126,7 +126,7 @@ func (s *AgentJobService) LoadTasksFromFile() error {
defer s.fileMutex.Unlock() defer s.fileMutex.Unlock()
if _, err := os.Stat(s.tasksFile); os.IsNotExist(err) { if _, err := os.Stat(s.tasksFile); os.IsNotExist(err) {
log.Debug().Msg("agent_tasks.json not found, starting with empty tasks") xlog.Debug("agent_tasks.json not found, starting with empty tasks")
return nil return nil
} }
@@ -145,12 +145,12 @@ func (s *AgentJobService) LoadTasksFromFile() error {
// Schedule cron if enabled and has cron expression // Schedule cron if enabled and has cron expression
if task.Enabled && task.Cron != "" { if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil { if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", task.ID).Msg("Failed to schedule cron task on load") xlog.Warn("Failed to schedule cron task on load", "error", err, "task_id", task.ID)
} }
} }
} }
log.Info().Int("count", len(tasksFile.Tasks)).Msg("Loaded tasks from file") xlog.Info("Loaded tasks from file", "count", len(tasksFile.Tasks))
return nil return nil
} }
@@ -190,7 +190,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
defer s.fileMutex.Unlock() defer s.fileMutex.Unlock()
if _, err := os.Stat(s.jobsFile); os.IsNotExist(err) { if _, err := os.Stat(s.jobsFile); os.IsNotExist(err) {
log.Debug().Msg("agent_jobs.json not found, starting with empty jobs") xlog.Debug("agent_jobs.json not found, starting with empty jobs")
return nil return nil
} }
@@ -209,7 +209,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
s.jobs.Set(job.ID, job) s.jobs.Set(job.ID, job)
} }
log.Info().Int("count", len(jobsFile.Jobs)).Msg("Loaded jobs from file") xlog.Info("Loaded jobs from file", "count", len(jobsFile.Jobs))
return nil return nil
} }
@@ -267,14 +267,14 @@ func (s *AgentJobService) CreateTask(task schema.Task) (string, error) {
// Schedule cron if enabled and has cron expression // Schedule cron if enabled and has cron expression
if task.Enabled && task.Cron != "" { if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil { if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task") xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
// Don't fail task creation if cron scheduling fails // Don't fail task creation if cron scheduling fails
} }
} }
// Save to file // Save to file
if err := s.SaveTasksToFile(); err != nil { if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file") xlog.Error("Failed to save tasks to file", "error", err)
// Don't fail task creation if file save fails // Don't fail task creation if file save fails
} }
@@ -304,13 +304,13 @@ func (s *AgentJobService) UpdateTask(id string, task schema.Task) error {
// Schedule new cron if enabled and has cron expression // Schedule new cron if enabled and has cron expression
if task.Enabled && task.Cron != "" { if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil { if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task") xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
} }
} }
// Save to file // Save to file
if err := s.SaveTasksToFile(); err != nil { if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file") xlog.Error("Failed to save tasks to file", "error", err)
} }
return nil return nil
@@ -330,7 +330,7 @@ func (s *AgentJobService) DeleteTask(id string) error {
// Save to file // Save to file
if err := s.SaveTasksToFile(); err != nil { if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file") xlog.Error("Failed to save tasks to file", "error", err)
} }
return nil return nil
@@ -409,7 +409,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
// Fetch content from URL with custom headers // Fetch content from URL with custom headers
dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type) dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type)
if err != nil { if err != nil {
log.Warn().Err(err).Str("url", source.URL).Str("type", source.Type).Msg("Failed to fetch multimedia from task source") xlog.Warn("Failed to fetch multimedia from task source", "error", err, "url", source.URL, "type", source.Type)
continue continue
} }
@@ -449,7 +449,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
// Save to file (async, don't block) // Save to file (async, don't block)
go func() { go func() {
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file") xlog.Error("Failed to save jobs to file", "error", err)
} }
}() }()
@@ -544,7 +544,7 @@ func (s *AgentJobService) CancelJob(id string) error {
// Save to file (async) // Save to file (async)
go func() { go func() {
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file") xlog.Error("Failed to save jobs to file", "error", err)
} }
}() }()
@@ -561,7 +561,7 @@ func (s *AgentJobService) DeleteJob(id string) error {
// Save to file // Save to file
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file") xlog.Error("Failed to save jobs to file", "error", err)
} }
return nil return nil
@@ -750,7 +750,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Images) > 0 { if len(job.Images) > 0 {
images, err := s.convertToMultimediaContent(job.Images, JobImageType) images, err := s.convertToMultimediaContent(job.Images, JobImageType)
if err != nil { if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert images") xlog.Warn("Failed to convert images", "error", err, "job_id", job.ID)
} else { } else {
multimediaItems = append(multimediaItems, images...) multimediaItems = append(multimediaItems, images...)
} }
@@ -760,7 +760,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Videos) > 0 { if len(job.Videos) > 0 {
videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType) videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType)
if err != nil { if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert videos") xlog.Warn("Failed to convert videos", "error", err, "job_id", job.ID)
} else { } else {
multimediaItems = append(multimediaItems, videos...) multimediaItems = append(multimediaItems, videos...)
} }
@@ -770,7 +770,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Audios) > 0 { if len(job.Audios) > 0 {
audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType) audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType)
if err != nil { if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert audios") xlog.Warn("Failed to convert audios", "error", err, "job_id", job.ID)
} else { } else {
multimediaItems = append(multimediaItems, audios...) multimediaItems = append(multimediaItems, audios...)
} }
@@ -780,7 +780,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Files) > 0 { if len(job.Files) > 0 {
files, err := s.convertToMultimediaContent(job.Files, JobFileType) files, err := s.convertToMultimediaContent(job.Files, JobFileType)
if err != nil { if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert files") xlog.Warn("Failed to convert files", "error", err, "job_id", job.ID)
} else { } else {
multimediaItems = append(multimediaItems, files...) multimediaItems = append(multimediaItems, files...)
} }
@@ -817,7 +817,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
cogito.WithContext(ctx), cogito.WithContext(ctx),
cogito.WithMCPs(sessions...), cogito.WithMCPs(sessions...),
cogito.WithStatusCallback(func(status string) { cogito.WithStatusCallback(func(status string) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Status: %s", status) xlog.Debug("Status", "job_id", job.ID, "model", modelConfig.Name, "status", status)
// Store trace // Store trace
trace := schema.JobTrace{ trace := schema.JobTrace{
Type: "status", Type: "status",
@@ -828,7 +828,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
s.jobs.Set(job.ID, job) s.jobs.Set(job.ID, job)
}), }),
cogito.WithReasoningCallback(func(reasoning string) { cogito.WithReasoningCallback(func(reasoning string) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Reasoning: %s", reasoning) xlog.Debug("Reasoning", "job_id", job.ID, "model", modelConfig.Name, "reasoning", reasoning)
// Store trace // Store trace
trace := schema.JobTrace{ trace := schema.JobTrace{
Type: "reasoning", Type: "reasoning",
@@ -839,9 +839,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
s.jobs.Set(job.ID, job) s.jobs.Set(job.ID, job)
}), }),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision { cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name). xlog.Debug("Tool call", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).
Msg("Tool call")
// Store trace // Store trace
arguments := make(map[string]interface{}) arguments := make(map[string]interface{})
if t.Arguments != nil { if t.Arguments != nil {
@@ -861,9 +859,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
} }
}), }),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) { cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name). xlog.Debug("Tool call result", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).
Msg("Tool call result")
// Store trace // Store trace
arguments := make(map[string]interface{}) arguments := make(map[string]interface{})
// Convert ToolArguments to map via JSON marshaling // Convert ToolArguments to map via JSON marshaling
@@ -988,7 +984,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
// Save to file (async) // Save to file (async)
go func() { go func() {
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file") xlog.Error("Failed to save jobs to file", "error", err)
} }
}() }()
@@ -1023,7 +1019,7 @@ func (s *AgentJobService) worker(ctx context.Context) {
// Execute job // Execute job
err := s.executeJobInternal(exec.Job, exec.Task, exec.Ctx) err := s.executeJobInternal(exec.Job, exec.Task, exec.Ctx)
if err != nil { if err != nil {
log.Error().Err(err).Str("job_id", exec.Job.ID).Msg("Job execution failed") xlog.Error("Job execution failed", "error", err, "job_id", exec.Job.ID)
} }
// Clean up cancellation // Clean up cancellation
@@ -1051,7 +1047,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
// Multimedia will be fetched from task sources in ExecuteJob // Multimedia will be fetched from task sources in ExecuteJob
_, err := s.ExecuteJob(task.ID, cronParams, "cron", nil) _, err := s.ExecuteJob(task.ID, cronParams, "cron", nil)
if err != nil { if err != nil {
log.Error().Err(err).Str("task_id", task.ID).Msg("Failed to execute cron job") xlog.Error("Failed to execute cron job", "error", err, "task_id", task.ID)
} }
}) })
if err != nil { if err != nil {
@@ -1059,7 +1055,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
} }
s.cronEntries.Set(task.ID, entryID) s.cronEntries.Set(task.ID, entryID)
log.Info().Str("task_id", task.ID).Str("cron", cronExpr).Msg("Scheduled cron task") xlog.Info("Scheduled cron task", "task_id", task.ID, "cron", cronExpr)
return nil return nil
} }
@@ -1069,7 +1065,7 @@ func (s *AgentJobService) UnscheduleCronTask(taskID string) {
entryID := s.cronEntries.Get(taskID) entryID := s.cronEntries.Get(taskID)
s.cronScheduler.Remove(entryID) s.cronScheduler.Remove(entryID)
s.cronEntries.Delete(taskID) s.cronEntries.Delete(taskID)
log.Info().Str("task_id", taskID).Msg("Unscheduled cron task") xlog.Info("Unscheduled cron task", "task_id", taskID)
} }
} }
@@ -1082,7 +1078,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
return // No webhooks configured return // No webhooks configured
} }
log.Info().Str("job_id", job.ID).Int("webhook_count", len(webhookConfigs)).Msg("Sending webhooks") xlog.Info("Sending webhooks", "job_id", job.ID, "webhook_count", len(webhookConfigs))
// Send all webhooks concurrently and track results // Send all webhooks concurrently and track results
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -1140,7 +1136,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
// Save to file (async) // Save to file (async)
go func() { go func() {
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file") xlog.Error("Failed to save jobs to file", "error", err)
} }
}() }()
} }
@@ -1157,11 +1153,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
// Build payload // Build payload
payload, err := s.buildWebhookPayload(job, task, webhookConfig) payload, err := s.buildWebhookPayload(job, task, webhookConfig)
if err != nil { if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to build webhook payload") xlog.Error("Failed to build webhook payload", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("failed to build payload: %w", err) return fmt.Errorf("failed to build payload: %w", err)
} }
log.Debug().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Str("payload", string(payload)).Msg("Sending webhook") xlog.Debug("Sending webhook", "job_id", job.ID, "webhook_url", webhookConfig.URL, "payload", string(payload))
// Determine HTTP method (default to POST) // Determine HTTP method (default to POST)
method := webhookConfig.Method method := webhookConfig.Method
@@ -1172,7 +1168,7 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
// Create HTTP request // Create HTTP request
req, err := http.NewRequest(method, webhookConfig.URL, bytes.NewBuffer(payload)) req, err := http.NewRequest(method, webhookConfig.URL, bytes.NewBuffer(payload))
if err != nil { if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to create webhook request") xlog.Error("Failed to create webhook request", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("failed to create request: %w", err) return fmt.Errorf("failed to create request: %w", err)
} }
@@ -1186,11 +1182,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
client := &http.Client{Timeout: 30 * time.Second} client := &http.Client{Timeout: 30 * time.Second}
err = s.executeWithRetry(client, req) err = s.executeWithRetry(client, req)
if err != nil { if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivery failed") xlog.Error("Webhook delivery failed", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("webhook delivery failed: %w", err) return fmt.Errorf("webhook delivery failed: %w", err)
} }
log.Info().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivered successfully") xlog.Info("Webhook delivered successfully", "job_id", job.ID, "webhook_url", webhookConfig.URL)
return nil return nil
} }
@@ -1303,10 +1299,10 @@ func (s *AgentJobService) CleanupOldJobs() error {
} }
if removed > 0 { if removed > 0 {
log.Info().Int("removed", removed).Int("retention_days", s.retentionDays).Msg("Cleaned up old jobs") xlog.Info("Cleaned up old jobs", "removed", removed, "retention_days", s.retentionDays)
// Save to file // Save to file
if err := s.SaveJobsToFile(); err != nil { if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file after cleanup") xlog.Error("Failed to save jobs to file after cleanup", "error", err)
} }
} }
@@ -1327,10 +1323,10 @@ func (s *AgentJobService) Start(ctx context.Context) error {
// Load tasks and jobs from files // Load tasks and jobs from files
if err := s.LoadTasksFromFile(); err != nil { if err := s.LoadTasksFromFile(); err != nil {
log.Warn().Err(err).Msg("Failed to load tasks from file") xlog.Warn("Failed to load tasks from file", "error", err)
} }
if err := s.LoadJobsFromFile(); err != nil { if err := s.LoadJobsFromFile(); err != nil {
log.Warn().Err(err).Msg("Failed to load jobs from file") xlog.Warn("Failed to load jobs from file", "error", err)
} }
// Start cron scheduler // Start cron scheduler
@@ -1345,19 +1341,19 @@ func (s *AgentJobService) Start(ctx context.Context) error {
// Schedule daily cleanup at midnight // Schedule daily cleanup at midnight
_, err := s.cronScheduler.AddFunc("0 0 * * *", func() { _, err := s.cronScheduler.AddFunc("0 0 * * *", func() {
if err := s.CleanupOldJobs(); err != nil { if err := s.CleanupOldJobs(); err != nil {
log.Error().Err(err).Msg("Failed to cleanup old jobs") xlog.Error("Failed to cleanup old jobs", "error", err)
} }
}) })
if err != nil { if err != nil {
log.Warn().Err(err).Msg("Failed to schedule daily cleanup") xlog.Warn("Failed to schedule daily cleanup", "error", err)
} }
// Run initial cleanup // Run initial cleanup
if err := s.CleanupOldJobs(); err != nil { if err := s.CleanupOldJobs(); err != nil {
log.Warn().Err(err).Msg("Failed to run initial cleanup") xlog.Warn("Failed to run initial cleanup", "error", err)
} }
log.Info().Int("retention_days", s.retentionDays).Msg("AgentJobService started") xlog.Info("AgentJobService started", "retention_days", s.retentionDays)
return nil return nil
} }
@@ -1370,7 +1366,7 @@ func (s *AgentJobService) Stop() error {
if s.cronScheduler != nil { if s.cronScheduler != nil {
s.cronScheduler.Stop() s.cronScheduler.Stop()
} }
log.Info().Msg("AgentJobService stopped") xlog.Info("AgentJobService stopped")
return nil return nil
} }
@@ -1380,5 +1376,5 @@ func (s *AgentJobService) UpdateRetentionDays(days int) {
if days == 0 { if days == 0 {
s.retentionDays = 30 // Default s.retentionDays = 30 // Default
} }
log.Info().Int("retention_days", s.retentionDays).Msg("Updated agent job retention days") xlog.Info("Updated agent job retention days", "retention_days", s.retentionDays)
} }

230
core/services/backend_monitor.go
View File

@@ -1,115 +1,115 @@
package services package services
import ( import (
"context" "context"
"fmt" "fmt"
"strings" "strings"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
gopsutil "github.com/shirou/gopsutil/v3/process" gopsutil "github.com/shirou/gopsutil/v3/process"
) )
type BackendMonitorService struct { type BackendMonitorService struct {
modelConfigLoader *config.ModelConfigLoader modelConfigLoader *config.ModelConfigLoader
modelLoader *model.ModelLoader modelLoader *model.ModelLoader
options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name. options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name.
} }
func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService { func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService {
return &BackendMonitorService{ return &BackendMonitorService{
modelLoader: modelLoader, modelLoader: modelLoader,
modelConfigLoader: configLoader, modelConfigLoader: configLoader,
options: appConfig, options: appConfig,
} }
} }
func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*schema.BackendMonitorResponse, error) { func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*schema.BackendMonitorResponse, error) {
config, exists := bms.modelConfigLoader.GetModelConfig(model) config, exists := bms.modelConfigLoader.GetModelConfig(model)
var backend string var backend string
if exists { if exists {
backend = config.Model backend = config.Model
} else { } else {
// Last ditch effort: use it raw, see if a backend happens to match. // Last ditch effort: use it raw, see if a backend happens to match.
backend = model backend = model
} }
if !strings.HasSuffix(backend, ".bin") { if !strings.HasSuffix(backend, ".bin") {
backend = fmt.Sprintf("%s.bin", backend) backend = fmt.Sprintf("%s.bin", backend)
} }
pid, err := bms.modelLoader.GetGRPCPID(backend) pid, err := bms.modelLoader.GetGRPCPID(backend)
if err != nil { if err != nil {
log.Error().Err(err).Str("model", model).Msg("failed to find GRPC pid") xlog.Error("failed to find GRPC pid", "error", err, "model", model)
return nil, err return nil, err
} }
// Name is slightly frightening but this does _not_ create a new process, rather it looks up an existing process by PID. // Name is slightly frightening but this does _not_ create a new process, rather it looks up an existing process by PID.
backendProcess, err := gopsutil.NewProcess(int32(pid)) backendProcess, err := gopsutil.NewProcess(int32(pid))
if err != nil { if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting process info") xlog.Error("error getting process info", "error", err, "model", model, "pid", pid)
return nil, err return nil, err
} }
memInfo, err := backendProcess.MemoryInfo() memInfo, err := backendProcess.MemoryInfo()
if err != nil { if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory info") xlog.Error("error getting memory info", "error", err, "model", model, "pid", pid)
return nil, err return nil, err
} }
memPercent, err := backendProcess.MemoryPercent() memPercent, err := backendProcess.MemoryPercent()
if err != nil { if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory percent") xlog.Error("error getting memory percent", "error", err, "model", model, "pid", pid)
return nil, err return nil, err
} }
cpuPercent, err := backendProcess.CPUPercent() cpuPercent, err := backendProcess.CPUPercent()
if err != nil { if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting cpu percent") xlog.Error("error getting cpu percent", "error", err, "model", model, "pid", pid)
return nil, err return nil, err
} }
return &schema.BackendMonitorResponse{ return &schema.BackendMonitorResponse{
MemoryInfo: memInfo, MemoryInfo: memInfo,
MemoryPercent: memPercent, MemoryPercent: memPercent,
CPUPercent: cpuPercent, CPUPercent: cpuPercent,
}, nil }, nil
} }
func (bms BackendMonitorService) CheckAndSample(modelName string) (*proto.StatusResponse, error) { func (bms BackendMonitorService) CheckAndSample(modelName string) (*proto.StatusResponse, error) {
modelAddr := bms.modelLoader.CheckIsLoaded(modelName) modelAddr := bms.modelLoader.CheckIsLoaded(modelName)
if modelAddr == nil { if modelAddr == nil {
return nil, fmt.Errorf("backend %s is not currently loaded", modelName) return nil, fmt.Errorf("backend %s is not currently loaded", modelName)
} }
status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO()) status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO())
if rpcErr != nil { if rpcErr != nil {
log.Warn().Msgf("backend %s experienced an error retrieving status info: %s", modelName, rpcErr.Error()) xlog.Warn("backend experienced an error retrieving status info", "backend", modelName, "error", rpcErr)
val, slbErr := bms.SampleLocalBackendProcess(modelName) val, slbErr := bms.SampleLocalBackendProcess(modelName)
if slbErr != nil { if slbErr != nil {
return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error()) return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error())
} }
return &proto.StatusResponse{ return &proto.StatusResponse{
State: proto.StatusResponse_ERROR, State: proto.StatusResponse_ERROR,
Memory: &proto.MemoryUsageData{ Memory: &proto.MemoryUsageData{
Total: val.MemoryInfo.VMS, Total: val.MemoryInfo.VMS,
Breakdown: map[string]uint64{ Breakdown: map[string]uint64{
"gopsutil-RSS": val.MemoryInfo.RSS, "gopsutil-RSS": val.MemoryInfo.RSS,
}, },
}, },
}, nil }, nil
} }
return status, nil return status, nil
} }
func (bms BackendMonitorService) ShutdownModel(modelName string) error { func (bms BackendMonitorService) ShutdownModel(modelName string) error {
return bms.modelLoader.ShutdownModel(modelName) return bms.modelLoader.ShutdownModel(modelName)
} }

16
core/services/backends.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, any], systemState *system.SystemState) error { func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, any], systemState *system.SystemState) error {
@@ -62,7 +62,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
g.modelLoader.DeleteExternalBackend(op.GalleryElementName) g.modelLoader.DeleteExternalBackend(op.GalleryElementName)
} else if op.ExternalURI != "" { } else if op.ExternalURI != "" {
// External backend installation (OCI image, URL, or path) // External backend installation (OCI image, URL, or path)
log.Info().Str("uri", op.ExternalURI).Str("name", op.ExternalName).Str("alias", op.ExternalAlias).Msg("Installing external backend") xlog.Info("Installing external backend", "uri", op.ExternalURI, "name", op.ExternalName, "alias", op.ExternalAlias)
err = InstallExternalBackend(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, progressCallback, op.ExternalURI, op.ExternalName, op.ExternalAlias) err = InstallExternalBackend(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, progressCallback, op.ExternalURI, op.ExternalName, op.ExternalAlias)
// Update GalleryElementName for status tracking if a name was derived // Update GalleryElementName for status tracking if a name was derived
if op.ExternalName != "" { if op.ExternalName != "" {
@@ -70,8 +70,8 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
} }
} else { } else {
// Standard gallery installation // Standard gallery installation
log.Warn().Msgf("installing backend %s", op.GalleryElementName) xlog.Warn("installing backend", "backend", op.GalleryElementName)
log.Debug().Msgf("backend galleries: %v", g.appConfig.BackendGalleries) xlog.Debug("backend galleries", "galleries", g.appConfig.BackendGalleries)
err = gallery.InstallBackendFromGallery(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true) err = gallery.InstallBackendFromGallery(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true)
} }
if err != nil { if err != nil {
@@ -85,7 +85,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
}) })
return err return err
} }
log.Error().Err(err).Msgf("error installing backend %s", op.GalleryElementName) xlog.Error("error installing backend", "error", err, "backend", op.GalleryElementName)
if !op.Delete { if !op.Delete {
// If we didn't install the backend, we need to make sure we don't have a leftover directory // If we didn't install the backend, we need to make sure we don't have a leftover directory
gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName) gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName)
@@ -114,7 +114,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
if name == "" { // infer it from the path if name == "" { // infer it from the path
name = filepath.Base(backend) name = filepath.Base(backend)
} }
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from path") xlog.Info("Installing backend from path", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{ if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{ Metadata: gallery.Metadata{
Name: name, Name: name,
@@ -128,7 +128,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
if name == "" { if name == "" {
return fmt.Errorf("specifying a name is required for OCI images") return fmt.Errorf("specifying a name is required for OCI images")
} }
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image") xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{ if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{ Metadata: gallery.Metadata{
Name: name, Name: name,
@@ -150,7 +150,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
name = derivedName name = derivedName
} }
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image") xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{ if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{ Metadata: gallery.Metadata{
Name: name, Name: name,

4
core/services/metrics.go
View File

@@ -3,7 +3,7 @@ package services
import ( import (
"context" "context"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/prometheus" "go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric"
@@ -49,6 +49,6 @@ func (lams LocalAIMetricsService) Shutdown() error {
//// setupOTelSDK bootstraps the OpenTelemetry pipeline. //// setupOTelSDK bootstraps the OpenTelemetry pipeline.
//// If it does not return an error, make sure to call shutdown for proper cleanup. //// If it does not return an error, make sure to call shutdown for proper cleanup.
log.Warn().Msgf("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?") xlog.Warn("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?")
return nil return nil
} }

4
core/services/models.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
@@ -208,7 +208,7 @@ func processModelOperation(
return err return err
} }
if automaticallyInstallBackend && installedModel.Backend != "" { if automaticallyInstallBackend && installedModel.Backend != "" {
log.Debug().Msgf("Installing backend %q", installedModel.Backend) xlog.Debug("Installing backend", "backend", installedModel.Backend)
if err := gallery.InstallBackendFromGallery(ctx, op.BackendGalleries, systemState, modelLoader, installedModel.Backend, progressCallback, false); err != nil { if err := gallery.InstallBackendFromGallery(ctx, op.BackendGalleries, systemState, modelLoader, installedModel.Backend, progressCallback, false); err != nil {
return err return err
} }

14
core/startup/model_preload.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -32,10 +32,10 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
// Check if it's a model gallery, or print a warning // Check if it's a model gallery, or print a warning
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries) e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries)
if e != nil && found { if e != nil && found {
log.Error().Err(err).Msgf("[startup] failed installing model '%s'", url) xlog.Error("[startup] failed installing model", "error", err, "model", url)
err = errors.Join(err, e) err = errors.Join(err, e)
} else if !found { } else if !found {
log.Debug().Msgf("[startup] model not found in the gallery '%s'", url) xlog.Debug("[startup] model not found in the gallery", "model", url)
if galleryService == nil { if galleryService == nil {
return fmt.Errorf("cannot start autoimporter, not sure how to handle this uri") return fmt.Errorf("cannot start autoimporter, not sure how to handle this uri")
@@ -44,7 +44,7 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
// TODO: we should just use the discoverModelConfig here and default to this. // TODO: we should just use the discoverModelConfig here and default to this.
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{}) modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
if discoverErr != nil { if discoverErr != nil {
log.Error().Err(discoverErr).Msgf("[startup] failed to discover model config '%s'", url) xlog.Error("[startup] failed to discover model config", "error", discoverErr, "model", url)
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err)) err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
continue continue
} }
@@ -76,11 +76,11 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
} }
if status.Error != nil { if status.Error != nil {
log.Error().Err(status.Error).Msgf("[startup] failed to import model '%s' from '%s'", modelConfig.Name, url) xlog.Error("[startup] failed to import model", "error", status.Error, "model", modelConfig.Name, "url", url)
return status.Error return status.Error
} }
log.Info().Msgf("[startup] imported model '%s' from '%s'", modelConfig.Name, url) xlog.Info("[startup] imported model", "model", modelConfig.Name, "url", url)
} }
} }
return err return err
@@ -101,7 +101,7 @@ func installModel(ctx context.Context, galleries, backendGalleries []config.Gall
downloadStatus = utils.DisplayDownloadFunction downloadStatus = utils.DisplayDownloadFunction
} }
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model") xlog.Info("installing model", "model", modelName, "license", model.License)
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries) err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries)
if err != nil { if err != nil {
return err, true return err, true

14
core/templates/evaluator.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/functions"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// Rather than pass an interface{} to the prompt template: // Rather than pass an interface{} to the prompt template:
@@ -133,13 +133,13 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
} }
templatedChatMessage, err := e.evaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData) templatedChatMessage, err := e.evaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData)
if err != nil { if err != nil {
log.Error().Err(err).Interface("message", chatMessageData).Str("template", config.TemplateConfig.ChatMessage).Msg("error processing message with template, skipping") xlog.Error("error processing message with template, skipping", "error", err, "message", chatMessageData, "template", config.TemplateConfig.ChatMessage)
} else { } else {
if templatedChatMessage == "" { if templatedChatMessage == "" {
log.Warn().Msgf("template \"%s\" produced blank output for %+v. Skipping!", config.TemplateConfig.ChatMessage, chatMessageData) xlog.Warn("template produced blank output, skipping", "template", config.TemplateConfig.ChatMessage, "message", chatMessageData)
continue // TODO: This continue is here intentionally to skip over the line `mess = append(mess, content)` below, and to prevent the sprintf continue // TODO: This continue is here intentionally to skip over the line `mess = append(mess, content)` below, and to prevent the sprintf
} }
log.Debug().Msgf("templated message for chat: %s", templatedChatMessage) xlog.Debug("templated message for chat", "message", templatedChatMessage)
content = templatedChatMessage content = templatedChatMessage
} }
} }
@@ -203,7 +203,7 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
} }
predInput = strings.Join(mess, joinCharacter) predInput = strings.Join(mess, joinCharacter)
log.Debug().Msgf("Prompt (before templating): %s", predInput) xlog.Debug("Prompt (before templating)", "prompt", predInput)
promptTemplate := ChatPromptTemplate promptTemplate := ChatPromptTemplate
@@ -221,9 +221,9 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
}) })
if err == nil { if err == nil {
predInput = templatedInput predInput = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", predInput) xlog.Debug("Template found, input modified", "input", predInput)
} else { } else {
log.Debug().Msgf("Template failed loading: %s", err.Error()) xlog.Debug("Template failed loading", "error", err)
} }
return predInput return predInput

2
go.mod
View File

@@ -41,7 +41,7 @@ require (
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_golang v1.23.2
github.com/robfig/cron/v3 v3.0.1 github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.34.0 github.com/mudler/xlog v0.0.2
github.com/russross/blackfriday v1.6.0 github.com/russross/blackfriday v1.6.0
github.com/sashabaranov/go-openai v1.41.2 github.com/sashabaranov/go-openai v1.41.2
github.com/schollz/progressbar/v3 v3.18.0 github.com/schollz/progressbar/v3 v3.18.0

2
go.sum
View File

@@ -518,6 +518,8 @@ github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2 h1:+WHsL/j6EWOMUiMVI
github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2/go.mod h1:EA8Ashhd56o32qN7ouPKFSRUs/Z+LrRCF4v6R2Oarm8= github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2/go.mod h1:EA8Ashhd56o32qN7ouPKFSRUs/Z+LrRCF4v6R2Oarm8=
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025 h1:WFLP5FHInarYGXi6B/Ze204x7Xy6q/I4nCZnWEyPHK0= github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025 h1:WFLP5FHInarYGXi6B/Ze204x7Xy6q/I4nCZnWEyPHK0=
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025/go.mod h1:QuIFdRstyGJt+MTTkWY+mtD7U6xwjOR6SwKUjmLZtR4= github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025/go.mod h1:QuIFdRstyGJt+MTTkWY+mtD7U6xwjOR6SwKUjmLZtR4=
github.com/mudler/xlog v0.0.2 h1:w3Z3HIexjYduveotEMlHAeKjTFDi15NzZLBtfV76i1Y=
github.com/mudler/xlog v0.0.2/go.mod h1:39f5vcd05Qd6GWKM8IjyHNQ7AmOx3ZM0YfhfIGhC18U=
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=

28
pkg/downloader/uri.go
View File

@@ -20,7 +20,7 @@ import (
"github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/oci"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/LocalAI/pkg/xio" "github.com/mudler/LocalAI/pkg/xio"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -70,7 +70,7 @@ func (uri URI) ReadWithAuthorizationAndCallback(ctx context.Context, basePath st
// Check if the local file is rooted in basePath // Check if the local file is rooted in basePath
err = utils.InTrustedRoot(resolvedFile, resolvedBasePath) err = utils.InTrustedRoot(resolvedFile, resolvedBasePath)
if err != nil { if err != nil {
log.Debug().Str("resolvedFile", resolvedFile).Str("basePath", basePath).Msg("downloader.GetURI blocked an attempt to ready a file url outside of basePath") xlog.Debug("downloader.GetURI blocked an attempt to ready a file url outside of basePath", "resolvedFile", resolvedFile, "basePath", basePath)
return err return err
} }
// Read the response body // Read the response body
@@ -245,11 +245,11 @@ func (s URI) ResolveURL() string {
func removePartialFile(tmpFilePath string) error { func removePartialFile(tmpFilePath string) error {
_, err := os.Stat(tmpFilePath) _, err := os.Stat(tmpFilePath)
if err == nil { if err == nil {
log.Debug().Msgf("Removing temporary file %s", tmpFilePath) xlog.Debug("Removing temporary file", "file", tmpFilePath)
err = os.Remove(tmpFilePath) err = os.Remove(tmpFilePath)
if err != nil { if err != nil {
err1 := fmt.Errorf("failed to remove temporary download file %s: %v", tmpFilePath, err) err1 := fmt.Errorf("failed to remove temporary download file %s: %v", tmpFilePath, err)
log.Warn().Msg(err1.Error()) xlog.Warn("failed to remove temporary download file", "error", err1)
return err1 return err1
} }
} }
@@ -333,7 +333,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// Check if the file already exists // Check if the file already exists
_, err := os.Stat(filePath) _, err := os.Stat(filePath)
if err == nil { if err == nil {
log.Debug().Str("filePath", filePath).Msg("[downloader] File already exists") xlog.Debug("[downloader] File already exists", "filePath", filePath)
// File exists, check SHA // File exists, check SHA
if sha != "" { if sha != "" {
// Verify SHA // Verify SHA
@@ -343,7 +343,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
} }
if calculatedSHA == sha { if calculatedSHA == sha {
// SHA matches, skip downloading // SHA matches, skip downloading
log.Debug().Msgf("File %q already exists and matches the SHA. Skipping download", filePath) xlog.Debug("File already exists and matches the SHA. Skipping download", "file", filePath)
return nil return nil
} }
// SHA doesn't match, delete the file and download again // SHA doesn't match, delete the file and download again
@@ -351,11 +351,11 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
if err != nil { if err != nil {
return fmt.Errorf("failed to remove existing file %q: %v", filePath, err) return fmt.Errorf("failed to remove existing file %q: %v", filePath, err)
} }
log.Debug().Msgf("Removed %q (SHA doesn't match)", filePath) xlog.Debug("Removed file (SHA doesn't match)", "file", filePath)
} else { } else {
// SHA is missing, skip downloading // SHA is missing, skip downloading
log.Debug().Msgf("File %q already exists. Skipping download", filePath) xlog.Debug("File already exists. Skipping download", "file", filePath)
return nil return nil
} }
} else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() { } else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() {
@@ -363,7 +363,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url) return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url)
} }
log.Info().Msgf("Downloading %s", url) xlog.Info("Downloading", "url", url)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil) req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil { if err != nil {
@@ -480,19 +480,19 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// Verify SHA // Verify SHA
calculatedSHA := fmt.Sprintf("%x", progress.hash.Sum(nil)) calculatedSHA := fmt.Sprintf("%x", progress.hash.Sum(nil))
if calculatedSHA != sha { if calculatedSHA != sha {
log.Debug().Msgf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha) xlog.Debug("SHA mismatch for file", "file", filePath, "calculated", calculatedSHA, "metadata", sha)
return fmt.Errorf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha) return fmt.Errorf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
} }
} else { } else {
log.Debug().Msgf("SHA missing for %q. Skipping validation", filePath) xlog.Debug("SHA missing. Skipping validation", "file", filePath)
} }
log.Info().Msgf("File %q downloaded and verified", filePath) xlog.Info("File downloaded and verified", "file", filePath)
if utils.IsArchive(filePath) { if utils.IsArchive(filePath) {
basePath := filepath.Dir(filePath) basePath := filepath.Dir(filePath)
log.Info().Msgf("File %q is an archive, uncompressing to %s", filePath, basePath) xlog.Info("File is an archive, uncompressing", "file", filePath, "basePath", basePath)
if err := utils.ExtractArchive(filePath, basePath); err != nil { if err := utils.ExtractArchive(filePath, basePath); err != nil {
log.Debug().Msgf("Failed decompressing %q: %s", filePath, err.Error()) xlog.Debug("Failed decompressing", "file", filePath, "error", err)
return err return err
} }
} }

6
pkg/functions/functions.go
View File

@@ -3,7 +3,7 @@ package functions
import ( import (
"encoding/json" "encoding/json"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -59,11 +59,11 @@ func (f Functions) ToJSONStructure(name, args string) JSONFunctionStructure {
err := json.Unmarshal(dat, &prop) err := json.Unmarshal(dat, &prop)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error unmarshalling dat") xlog.Error("error unmarshalling dat", "error", err)
} }
err = json.Unmarshal(dat2, &defsD) err = json.Unmarshal(dat2, &defsD)
if err != nil { if err != nil {
log.Error().Err(err).Msg("error unmarshalling dat2") xlog.Error("error unmarshalling dat2", "error", err)
} }
if js.Defs == nil { if js.Defs == nil {
js.Defs = defsD js.Defs = defsD

22
pkg/functions/parse.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/pkg/functions/grammars" "github.com/mudler/LocalAI/pkg/functions/grammars"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// @Description GrammarConfig contains configuration for grammar parsing // @Description GrammarConfig contains configuration for grammar parsing
@@ -150,22 +150,22 @@ func (g FunctionsConfig) GrammarOptions() []func(o *grammars.GrammarOption) {
} }
func CleanupLLMResult(llmresult string, functionConfig FunctionsConfig) string { func CleanupLLMResult(llmresult string, functionConfig FunctionsConfig) string {
log.Debug().Msgf("LLM result: %s", llmresult) xlog.Debug("LLM result", "result", llmresult)
for _, item := range functionConfig.ReplaceLLMResult { for _, item := range functionConfig.ReplaceLLMResult {
k, v := item.Key, item.Value k, v := item.Key, item.Value
log.Debug().Msgf("Replacing %s with %s", k, v) xlog.Debug("Replacing", "key", k, "value", v)
re := regexp.MustCompile(k) re := regexp.MustCompile(k)
llmresult = re.ReplaceAllString(llmresult, v) llmresult = re.ReplaceAllString(llmresult, v)
} }
log.Debug().Msgf("LLM result(processed): %s", llmresult) xlog.Debug("LLM result(processed)", "result", llmresult)
return llmresult return llmresult
} }
func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string { func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string {
log.Debug().Msgf("ParseTextContent: %s", llmresult) xlog.Debug("ParseTextContent", "result", llmresult)
log.Debug().Msgf("CaptureLLMResult: %s", functionConfig.CaptureLLMResult) xlog.Debug("CaptureLLMResult", "config", functionConfig.CaptureLLMResult)
for _, r := range functionConfig.CaptureLLMResult { for _, r := range functionConfig.CaptureLLMResult {
// We use a regex to extract the JSON object from the response // We use a regex to extract the JSON object from the response
@@ -223,15 +223,15 @@ func ParseJSON(s string) ([]map[string]any, error) {
func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults { func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults {
log.Debug().Msgf("LLM result: %s", llmresult) xlog.Debug("LLM result", "result", llmresult)
for _, item := range functionConfig.ReplaceFunctionResults { for _, item := range functionConfig.ReplaceFunctionResults {
k, v := item.Key, item.Value k, v := item.Key, item.Value
log.Debug().Msgf("Replacing %s with %s", k, v) xlog.Debug("Replacing", "key", k, "value", v)
re := regexp.MustCompile(k) re := regexp.MustCompile(k)
llmresult = re.ReplaceAllString(llmresult, v) llmresult = re.ReplaceAllString(llmresult, v)
} }
log.Debug().Msgf("LLM result(function cleanup): %s", llmresult) xlog.Debug("LLM result(function cleanup)", "result", llmresult)
functionNameKey := defaultFunctionNameKey functionNameKey := defaultFunctionNameKey
functionArgumentsKey := defaultFunctionArgumentsKey functionArgumentsKey := defaultFunctionArgumentsKey
@@ -256,10 +256,10 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
ss, err := ParseJSON(s) ss, err := ParseJSON(s)
//err := json.Unmarshal([]byte(s), &ss) //err := json.Unmarshal([]byte(s), &ss)
if err != nil { if err != nil {
log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects") xlog.Debug("unable to unmarshal llm result in a single object or an array of JSON objects", "error", err, "escapedLLMResult", s)
} }
log.Debug().Msgf("Function return: %s %+v", s, ss) xlog.Debug("Function return", "result", s, "parsed", ss)
for _, s := range ss { for _, s := range ss {
// The grammar defines the function name as "function", while OpenAI returns "name" // The grammar defines the function name as "function", while OpenAI returns "name"

56
pkg/model/initializers.go
View File

@@ -10,7 +10,7 @@ import (
grpc "github.com/mudler/LocalAI/pkg/grpc" grpc "github.com/mudler/LocalAI/pkg/grpc"
"github.com/phayes/freeport" "github.com/phayes/freeport"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -50,7 +50,7 @@ const (
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) { func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
return func(modelID, modelName, modelFile string) (*Model, error) { return func(modelID, modelName, modelFile string) (*Model, error) {
log.Debug().Msgf("Loading Model %s with gRPC (file: %s) (backend: %s): %+v", modelID, modelFile, backend, *o) xlog.Debug("Loading Model with gRPC", "modelID", modelID, "file", modelFile, "backend", backend, "options", *o)
var client *Model var client *Model
@@ -67,17 +67,17 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
if os.Getenv(env) == "" { if os.Getenv(env) == "" {
err := os.Setenv(env, ml.ModelPath) err := os.Setenv(env, ml.ModelPath)
if err != nil { if err != nil {
log.Error().Err(err).Str("name", env).Str("modelPath", ml.ModelPath).Msg("unable to set environment variable to modelPath") xlog.Error("unable to set environment variable to modelPath", "error", err, "name", env, "modelPath", ml.ModelPath)
} }
} }
} }
// Check if the backend is provided as external // Check if the backend is provided as external
if uri, ok := ml.GetAllExternalBackends(o)[backend]; ok { if uri, ok := ml.GetAllExternalBackends(o)[backend]; ok {
log.Debug().Msgf("Loading external backend: %s", uri) xlog.Debug("Loading external backend", "uri", uri)
// check if uri is a file or a address // check if uri is a file or a address
if fi, err := os.Stat(uri); err == nil { if fi, err := os.Stat(uri); err == nil {
log.Debug().Msgf("external backend is file: %+v", fi) xlog.Debug("external backend is file", "file", fi)
serverAddress, err := getFreeAddress() serverAddress, err := getFreeAddress()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error()) return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
@@ -85,43 +85,43 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
// Make sure the process is executable // Make sure the process is executable
process, err := ml.startProcess(uri, modelID, serverAddress) process, err := ml.startProcess(uri, modelID, serverAddress)
if err != nil { if err != nil {
log.Error().Err(err).Str("path", uri).Msg("failed to launch ") xlog.Error("failed to launch", "error", err, "path", uri)
return nil, err return nil, err
} }
log.Debug().Msgf("GRPC Service Started") xlog.Debug("GRPC Service Started")
client = NewModel(modelID, serverAddress, process) client = NewModel(modelID, serverAddress, process)
} else { } else {
log.Debug().Msg("external backend is a uri") xlog.Debug("external backend is a uri")
// address // address
client = NewModel(modelID, uri, nil) client = NewModel(modelID, uri, nil)
} }
} else { } else {
log.Error().Msgf("Backend not found: %s", backend) xlog.Error("Backend not found", "backend", backend)
return nil, fmt.Errorf("backend not found: %s", backend) return nil, fmt.Errorf("backend not found: %s", backend)
} }
log.Debug().Msgf("Wait for the service to start up") xlog.Debug("Wait for the service to start up")
log.Debug().Msgf("Options: %+v", o.gRPCOptions) xlog.Debug("Options", "options", o.gRPCOptions)
// Wait for the service to start up // Wait for the service to start up
ready := false ready := false
for i := 0; i < o.grpcAttempts; i++ { for i := 0; i < o.grpcAttempts; i++ {
alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background()) alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background())
if alive { if alive {
log.Debug().Msgf("GRPC Service Ready") xlog.Debug("GRPC Service Ready")
ready = true ready = true
break break
} }
if err != nil && i == o.grpcAttempts-1 { if err != nil && i == o.grpcAttempts-1 {
log.Error().Err(err).Msg("failed starting/connecting to the gRPC service") xlog.Error("failed starting/connecting to the gRPC service", "error", err)
} }
time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second) time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second)
} }
if !ready { if !ready {
log.Debug().Msgf("GRPC Service NOT ready") xlog.Debug("GRPC Service NOT ready")
if process := client.Process(); process != nil { if process := client.Process(); process != nil {
process.Stop() process.Stop()
} }
@@ -133,7 +133,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
options.ModelFile = modelFile options.ModelFile = modelFile
options.ModelPath = ml.ModelPath options.ModelPath = ml.ModelPath
log.Debug().Msgf("GRPC: Loading model with options: %+v", options) xlog.Debug("GRPC: Loading model with options", "options", options)
res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options) res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options)
if err != nil { if err != nil {
@@ -156,16 +156,16 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) { func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
o := NewOptions(opts...) o := NewOptions(opts...)
log.Info().Str("modelID", o.modelID).Str("backend", o.backendString).Str("o.model", o.model).Msg("BackendLoader starting") xlog.Info("BackendLoader starting", "modelID", o.modelID, "backend", o.backendString, "model", o.model)
backend := strings.ToLower(o.backendString) backend := strings.ToLower(o.backendString)
if realBackend, exists := Aliases[backend]; exists { if realBackend, exists := Aliases[backend]; exists {
typeAlias, exists := TypeAlias[backend] typeAlias, exists := TypeAlias[backend]
if exists { if exists {
log.Debug().Msgf("'%s' is a type alias of '%s' (%s)", backend, realBackend, typeAlias) xlog.Debug("alias is a type alias", "alias", backend, "realBackend", realBackend, "type", typeAlias)
o.gRPCOptions.Type = typeAlias o.gRPCOptions.Type = typeAlias
} else { } else {
log.Debug().Msgf("'%s' is an alias of '%s'", backend, realBackend) xlog.Debug("alias", "alias", backend, "realBackend", realBackend)
} }
backend = realBackend backend = realBackend
@@ -174,9 +174,9 @@ func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err e
model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o)) model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o))
if err != nil { if err != nil {
if stopErr := ml.StopGRPC(only(o.modelID));stopErr != nil { if stopErr := ml.StopGRPC(only(o.modelID));stopErr != nil {
log.Error().Err(stopErr).Str("model", o.modelID).Msg("error stopping model") xlog.Error("error stopping model", "error", stopErr, "model", o.modelID)
} }
log.Error().Str("modelID", o.modelID).Err(err).Msgf("Failed to load model %s with backend %s", o.modelID, o.backendString) xlog.Error("Failed to load model", "modelID", o.modelID, "error", err, "backend", o.backendString)
return nil, err return nil, err
} }
@@ -209,7 +209,7 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
// Return earlier if we have a model already loaded // Return earlier if we have a model already loaded
// (avoid looping through all the backends) // (avoid looping through all the backends)
if m := ml.CheckIsLoaded(o.modelID); m != nil { if m := ml.CheckIsLoaded(o.modelID); m != nil {
log.Debug().Msgf("Model '%s' already loaded", o.modelID) xlog.Debug("Model already loaded", "model", o.modelID)
// Update last used time for LRU tracking // Update last used time for LRU tracking
ml.updateModelLastUsed(m) ml.updateModelLastUsed(m)
return m.GRPC(o.parallelRequests, ml.wd), nil return m.GRPC(o.parallelRequests, ml.wd), nil
@@ -239,30 +239,30 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
} }
if len(autoLoadBackends) == 0 { if len(autoLoadBackends) == 0 {
log.Error().Msg("No backends found") xlog.Error("No backends found")
return nil, fmt.Errorf("no backends found") return nil, fmt.Errorf("no backends found")
} }
log.Debug().Msgf("Loading from the following backends (in order): %+v", autoLoadBackends) xlog.Debug("Loading from the following backends (in order)", "backends", autoLoadBackends)
log.Info().Msgf("Trying to load the model '%s' with the backend '%s'", o.modelID, autoLoadBackends) xlog.Info("Trying to load the model", "modelID", o.modelID, "backends", autoLoadBackends)
for _, key := range autoLoadBackends { for _, key := range autoLoadBackends {
log.Info().Msgf("[%s] Attempting to load", key) xlog.Info("Attempting to load", "backend", key)
options := append(opts, []Option{ options := append(opts, []Option{
WithBackendString(key), WithBackendString(key),
}...) }...)
model, modelerr := ml.backendLoader(options...) model, modelerr := ml.backendLoader(options...)
if modelerr == nil && model != nil { if modelerr == nil && model != nil {
log.Info().Msgf("[%s] Loads OK", key) xlog.Info("Loads OK", "backend", key)
return model, nil return model, nil
} else if modelerr != nil { } else if modelerr != nil {
err = errors.Join(err, fmt.Errorf("[%s]: %w", key, modelerr)) err = errors.Join(err, fmt.Errorf("[%s]: %w", key, modelerr))
log.Info().Msgf("[%s] Fails: %s", key, modelerr.Error()) xlog.Info("Fails", "backend", key, "error", modelerr.Error())
} else if model == nil { } else if model == nil {
err = errors.Join(err, fmt.Errorf("backend %s returned no usable model", key)) err = errors.Join(err, fmt.Errorf("backend %s returned no usable model", key))
log.Info().Msgf("[%s] Fails: %s", key, "backend returned no usable model") xlog.Info("Fails", "backend", key, "error", "backend returned no usable model")
} }
} }

20
pkg/model/loader.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils" "github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// new idea: what if we declare a struct of these here, and use a loop to check? // new idea: what if we declare a struct of these here, and use a loop to check?
@@ -173,7 +173,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
if loadingChan, isLoading := ml.loading[modelID]; isLoading { if loadingChan, isLoading := ml.loading[modelID]; isLoading {
ml.mu.Unlock() ml.mu.Unlock()
// Wait for the other goroutine to finish loading // Wait for the other goroutine to finish loading
log.Debug().Str("modelID", modelID).Msg("Waiting for model to be loaded by another request") xlog.Debug("Waiting for model to be loaded by another request", "modelID", modelID)
<-loadingChan <-loadingChan
// Now check if the model is loaded // Now check if the model is loaded
ml.mu.Lock() ml.mu.Lock()
@@ -201,7 +201,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
// Load the model (this can take a long time, no lock held) // Load the model (this can take a long time, no lock held)
modelFile := filepath.Join(ml.ModelPath, modelName) modelFile := filepath.Join(ml.ModelPath, modelName)
log.Debug().Msgf("Loading model in memory from file: %s", modelFile) xlog.Debug("Loading model in memory from file", "file", modelFile)
model, err := loader(modelID, modelName, modelFile) model, err := loader(modelID, modelName, modelFile)
if err != nil { if err != nil {
@@ -239,28 +239,28 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model {
return nil return nil
} }
log.Debug().Msgf("Model already loaded in memory: %s", s) xlog.Debug("Model already loaded in memory", "model", s)
client := m.GRPC(false, ml.wd) client := m.GRPC(false, ml.wd)
log.Debug().Msgf("Checking model availability (%s)", s) xlog.Debug("Checking model availability", "model", s)
cTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute) cTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel() defer cancel()
alive, err := client.HealthCheck(cTimeout) alive, err := client.HealthCheck(cTimeout)
if !alive { if !alive {
log.Warn().Msgf("GRPC Model not responding: %s", err.Error()) xlog.Warn("GRPC Model not responding", "error", err)
log.Warn().Msgf("Deleting the process in order to recreate it") xlog.Warn("Deleting the process in order to recreate it")
process := m.Process() process := m.Process()
if process == nil { if process == nil {
log.Error().Msgf("Process not found for '%s' and the model is not responding anymore !", s) xlog.Error("Process not found and the model is not responding anymore", "model", s)
return m return m
} }
if !process.IsAlive() { if !process.IsAlive() {
log.Debug().Msgf("GRPC Process is not responding: %s", s) xlog.Debug("GRPC Process is not responding", "model", s)
// stop and delete the process, this forces to re-load the model and re-create again the service // stop and delete the process, this forces to re-load the model and re-create again the service
err := ml.deleteProcess(s) err := ml.deleteProcess(s)
if err != nil { if err != nil {
log.Error().Err(err).Str("process", s).Msg("error stopping process") xlog.Error("error stopping process", "error", err, "process", s)
} }
return nil return nil
} }

32
pkg/model/process.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/hpcloud/tail" "github.com/hpcloud/tail"
"github.com/mudler/LocalAI/pkg/signals" "github.com/mudler/LocalAI/pkg/signals"
process "github.com/mudler/go-processmanager" process "github.com/mudler/go-processmanager"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true" var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true"
@@ -20,7 +20,7 @@ var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "
func (ml *ModelLoader) deleteProcess(s string) error { func (ml *ModelLoader) deleteProcess(s string) error {
model, ok := ml.models[s] model, ok := ml.models[s]
if !ok { if !ok {
log.Debug().Msgf("Model %s not found", s) xlog.Debug("Model not found", "model", s)
return fmt.Errorf("model %s not found", s) return fmt.Errorf("model %s not found", s)
} }
@@ -28,7 +28,7 @@ func (ml *ModelLoader) deleteProcess(s string) error {
retries := 1 retries := 1
for model.GRPC(false, ml.wd).IsBusy() { for model.GRPC(false, ml.wd).IsBusy() {
log.Debug().Msgf("%s busy. Waiting.", s) xlog.Debug("Model busy. Waiting.", "model", s)
dur := time.Duration(retries*2) * time.Second dur := time.Duration(retries*2) * time.Second
if dur > retryTimeout { if dur > retryTimeout {
dur = retryTimeout dur = retryTimeout
@@ -37,23 +37,23 @@ func (ml *ModelLoader) deleteProcess(s string) error {
retries++ retries++
if retries > 10 && forceBackendShutdown { if retries > 10 && forceBackendShutdown {
log.Warn().Msgf("Model %s is still busy after %d retries. Forcing shutdown.", s, retries) xlog.Warn("Model is still busy after retries. Forcing shutdown.", "model", s, "retries", retries)
break break
} }
} }
log.Debug().Msgf("Deleting process %s", s) xlog.Debug("Deleting process", "model", s)
process := model.Process() process := model.Process()
if process == nil { if process == nil {
log.Error().Msgf("No process for %s", s) xlog.Error("No process", "model", s)
// Nothing to do as there is no process // Nothing to do as there is no process
return nil return nil
} }
err := process.Stop() err := process.Stop()
if err != nil { if err != nil {
log.Error().Err(err).Msgf("(deleteProcess) error while deleting process %s", s) xlog.Error("(deleteProcess) error while deleting process", "error", err, "model", s)
} }
return err return err
@@ -95,16 +95,16 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
// Check first if it has executable permissions // Check first if it has executable permissions
if fi, err := os.Stat(grpcProcess); err == nil { if fi, err := os.Stat(grpcProcess); err == nil {
if fi.Mode()&0111 == 0 { if fi.Mode()&0111 == 0 {
log.Debug().Msgf("Process %s is not executable. Making it executable.", grpcProcess) xlog.Debug("Process is not executable. Making it executable.", "process", grpcProcess)
if err := os.Chmod(grpcProcess, 0700); err != nil { if err := os.Chmod(grpcProcess, 0700); err != nil {
return nil, err return nil, err
} }
} }
} }
log.Debug().Msgf("Loading GRPC Process: %s", grpcProcess) xlog.Debug("Loading GRPC Process", "process", grpcProcess)
log.Debug().Msgf("GRPC Service for %s will be running at: '%s'", id, serverAddress) xlog.Debug("GRPC Service will be running", "id", id, "address", serverAddress)
workDir, err := filepath.Abs(filepath.Dir(grpcProcess)) workDir, err := filepath.Abs(filepath.Dir(grpcProcess))
if err != nil { if err != nil {
@@ -128,31 +128,31 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
return grpcControlProcess, err return grpcControlProcess, err
} }
log.Debug().Msgf("GRPC Service state dir: %s", grpcControlProcess.StateDir()) xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir())
signals.RegisterGracefulTerminationHandler(func() { signals.RegisterGracefulTerminationHandler(func() {
err := grpcControlProcess.Stop() err := grpcControlProcess.Stop()
if err != nil { if err != nil {
log.Error().Err(err).Msg("error while shutting down grpc process") xlog.Error("error while shutting down grpc process", "error", err)
} }
}) })
go func() { go func() {
t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true}) t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true})
if err != nil { if err != nil {
log.Debug().Msgf("Could not tail stderr") xlog.Debug("Could not tail stderr")
} }
for line := range t.Lines { for line := range t.Lines {
log.Debug().Msgf("GRPC(%s): stderr %s", strings.Join([]string{id, serverAddress}, "-"), line.Text) xlog.Debug("GRPC stderr", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
} }
}() }()
go func() { go func() {
t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true}) t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true})
if err != nil { if err != nil {
log.Debug().Msgf("Could not tail stdout") xlog.Debug("Could not tail stdout")
} }
for line := range t.Lines { for line := range t.Lines {
log.Debug().Msgf("GRPC(%s): stdout %s", strings.Join([]string{id, serverAddress}, "-"), line.Text) xlog.Debug("GRPC stdout", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
} }
}() }()

66
pkg/model/watchdog.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
process "github.com/mudler/go-processmanager" process "github.com/mudler/go-processmanager"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// WatchDog tracks all the requests from GRPC clients. // WatchDog tracks all the requests from GRPC clients.
@@ -113,7 +113,7 @@ func (wd *WatchDog) GetMemoryReclaimerSettings() (enabled bool, threshold float6
func (wd *WatchDog) Shutdown() { func (wd *WatchDog) Shutdown() {
wd.Lock() wd.Lock()
defer wd.Unlock() defer wd.Unlock()
log.Info().Msg("[WatchDog] Shutting down watchdog") xlog.Info("[WatchDog] Shutting down watchdog")
wd.stop <- true wd.stop <- true
} }
@@ -191,7 +191,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
return 0 return 0
} }
log.Debug().Int("current", currentCount).Int("pendingLoads", pendingLoads).Int("limit", wd.lruLimit).Int("toEvict", modelsToEvict).Msg("[WatchDog] LRU enforcement triggered") xlog.Debug("[WatchDog] LRU enforcement triggered", "current", currentCount, "pendingLoads", pendingLoads, "limit", wd.lruLimit, "toEvict", modelsToEvict)
// Build a list of models sorted by last used time (oldest first) // Build a list of models sorted by last used time (oldest first)
var models []modelUsageInfo var models []modelUsageInfo
@@ -217,7 +217,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
var modelsToShutdown []string var modelsToShutdown []string
for i := 0; i < modelsToEvict && i < len(models); i++ { for i := 0; i < modelsToEvict && i < len(models); i++ {
m := models[i] m := models[i]
log.Info().Str("model", m.model).Time("lastUsed", m.lastUsed).Msg("[WatchDog] LRU evicting model") xlog.Info("[WatchDog] LRU evicting model", "model", m.model, "lastUsed", m.lastUsed)
modelsToShutdown = append(modelsToShutdown, m.model) modelsToShutdown = append(modelsToShutdown, m.model)
// Clean up the maps while we have the lock // Clean up the maps while we have the lock
wd.untrack(m.address) wd.untrack(m.address)
@@ -227,21 +227,21 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
// Now shutdown models without holding the watchdog lock to prevent deadlock // Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown { for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil { if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[WatchDog] error shutting down model during LRU eviction") xlog.Error("[WatchDog] error shutting down model during LRU eviction", "error", err, "model", model)
} }
log.Debug().Str("model", model).Msg("[WatchDog] LRU eviction complete") xlog.Debug("[WatchDog] LRU eviction complete", "model", model)
} }
return len(modelsToShutdown) return len(modelsToShutdown)
} }
func (wd *WatchDog) Run() { func (wd *WatchDog) Run() {
log.Info().Msg("[WatchDog] starting watchdog") xlog.Info("[WatchDog] starting watchdog")
for { for {
select { select {
case <-wd.stop: case <-wd.stop:
log.Info().Msg("[WatchDog] Stopping watchdog") xlog.Info("[WatchDog] Stopping watchdog")
return return
case <-time.After(wd.watchdogInterval): case <-time.After(wd.watchdogInterval):
// Check if any monitoring is enabled // Check if any monitoring is enabled
@@ -252,7 +252,7 @@ func (wd *WatchDog) Run() {
wd.Unlock() wd.Unlock()
if !busyCheck && !idleCheck && !memoryCheck { if !busyCheck && !idleCheck && !memoryCheck {
log.Info().Msg("[WatchDog] No checks enabled, stopping watchdog") xlog.Info("[WatchDog] No checks enabled, stopping watchdog")
return return
} }
if busyCheck { if busyCheck {
@@ -270,19 +270,19 @@ func (wd *WatchDog) Run() {
func (wd *WatchDog) checkIdle() { func (wd *WatchDog) checkIdle() {
wd.Lock() wd.Lock()
log.Debug().Msg("[WatchDog] Watchdog checks for idle connections") xlog.Debug("[WatchDog] Watchdog checks for idle connections")
// Collect models to shutdown while holding the lock // Collect models to shutdown while holding the lock
var modelsToShutdown []string var modelsToShutdown []string
for address, t := range wd.idleTime { for address, t := range wd.idleTime {
log.Debug().Msgf("[WatchDog] %s: idle connection", address) xlog.Debug("[WatchDog] idle connection", "address", address)
if time.Since(t) > wd.idletimeout { if time.Since(t) > wd.idletimeout {
log.Warn().Msgf("[WatchDog] Address %s is idle for too long, killing it", address) xlog.Warn("[WatchDog] Address is idle for too long, killing it", "address", address)
model, ok := wd.addressModelMap[address] model, ok := wd.addressModelMap[address]
if ok { if ok {
modelsToShutdown = append(modelsToShutdown, model) modelsToShutdown = append(modelsToShutdown, model)
} else { } else {
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address) xlog.Warn("[WatchDog] Address unresolvable", "address", address)
} }
wd.untrack(address) wd.untrack(address)
} }
@@ -292,28 +292,28 @@ func (wd *WatchDog) checkIdle() {
// Now shutdown models without holding the watchdog lock to prevent deadlock // Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown { for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil { if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model") xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
} }
log.Debug().Msgf("[WatchDog] model shut down: %s", model) xlog.Debug("[WatchDog] model shut down", "model", model)
} }
} }
func (wd *WatchDog) checkBusy() { func (wd *WatchDog) checkBusy() {
wd.Lock() wd.Lock()
log.Debug().Msg("[WatchDog] Watchdog checks for busy connections") xlog.Debug("[WatchDog] Watchdog checks for busy connections")
// Collect models to shutdown while holding the lock // Collect models to shutdown while holding the lock
var modelsToShutdown []string var modelsToShutdown []string
for address, t := range wd.busyTime { for address, t := range wd.busyTime {
log.Debug().Msgf("[WatchDog] %s: active connection", address) xlog.Debug("[WatchDog] active connection", "address", address)
if time.Since(t) > wd.timeout { if time.Since(t) > wd.timeout {
model, ok := wd.addressModelMap[address] model, ok := wd.addressModelMap[address]
if ok { if ok {
log.Warn().Msgf("[WatchDog] Model %s is busy for too long, killing it", model) xlog.Warn("[WatchDog] Model is busy for too long, killing it", "model", model)
modelsToShutdown = append(modelsToShutdown, model) modelsToShutdown = append(modelsToShutdown, model)
} else { } else {
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address) xlog.Warn("[WatchDog] Address unresolvable", "address", address)
} }
wd.untrack(address) wd.untrack(address)
} }
@@ -323,9 +323,9 @@ func (wd *WatchDog) checkBusy() {
// Now shutdown models without holding the watchdog lock to prevent deadlock // Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown { for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil { if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model") xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
} }
log.Debug().Msgf("[WatchDog] model shut down: %s", model) xlog.Debug("[WatchDog] model shut down", "model", model)
} }
} }
@@ -344,7 +344,7 @@ func (wd *WatchDog) checkMemory() {
// Get current memory usage (GPU if available, otherwise RAM) // Get current memory usage (GPU if available, otherwise RAM)
aggregate := xsysinfo.GetResourceAggregateInfo() aggregate := xsysinfo.GetResourceAggregateInfo()
if aggregate.TotalMemory == 0 { if aggregate.TotalMemory == 0 {
log.Debug().Msg("[WatchDog] No memory information available for memory reclaimer") xlog.Debug("[WatchDog] No memory information available for memory reclaimer")
return return
} }
@@ -356,20 +356,11 @@ func (wd *WatchDog) checkMemory() {
memoryType = "RAM" memoryType = "RAM"
} }
log.Debug(). xlog.Debug("[WatchDog] Memory check", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent, "loaded_models", modelCount)
Str("type", memoryType).
Float64("usage_percent", aggregate.UsagePercent).
Float64("threshold_percent", thresholdPercent).
Int("loaded_models", modelCount).
Msg("[WatchDog] Memory check")
// Check if usage exceeds threshold // Check if usage exceeds threshold
if aggregate.UsagePercent > thresholdPercent { if aggregate.UsagePercent > thresholdPercent {
log.Warn(). xlog.Warn("[WatchDog] Memory usage exceeds threshold, evicting LRU backend", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent)
Str("type", memoryType).
Float64("usage_percent", aggregate.UsagePercent).
Float64("threshold_percent", thresholdPercent).
Msg("[WatchDog] Memory usage exceeds threshold, evicting LRU backend")
// Evict the least recently used model // Evict the least recently used model
wd.evictLRUModel() wd.evictLRUModel()
@@ -411,10 +402,7 @@ func (wd *WatchDog) evictLRUModel() {
// Get the LRU model // Get the LRU model
lruModel := models[0] lruModel := models[0]
log.Info(). xlog.Info("[WatchDog] Memory reclaimer evicting LRU model", "model", lruModel.model, "lastUsed", lruModel.lastUsed)
Str("model", lruModel.model).
Time("lastUsed", lruModel.lastUsed).
Msg("[WatchDog] Memory reclaimer evicting LRU model")
// Untrack the model // Untrack the model
wd.untrack(lruModel.address) wd.untrack(lruModel.address)
@@ -422,9 +410,9 @@ func (wd *WatchDog) evictLRUModel() {
// Shutdown the model // Shutdown the model
if err := wd.pm.ShutdownModel(lruModel.model); err != nil { if err := wd.pm.ShutdownModel(lruModel.model); err != nil {
log.Error().Err(err).Str("model", lruModel.model).Msg("[WatchDog] error shutting down model during memory reclamation") xlog.Error("[WatchDog] error shutting down model during memory reclamation", "error", err, "model", lruModel.model)
} else { } else {
log.Info().Str("model", lruModel.model).Msg("[WatchDog] Memory reclaimer eviction complete") xlog.Info("[WatchDog] Memory reclaimer eviction complete", "model", lruModel.model)
} }
} }

22
pkg/system/capabilities.go
View File

@@ -9,7 +9,7 @@ import (
"strings" "strings"
"github.com/jaypipes/ghw/pkg/gpu" "github.com/jaypipes/ghw/pkg/gpu"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
const ( const (
@@ -49,11 +49,11 @@ func (s *SystemState) Capability(capMap map[string]string) string {
// Check if the reported capability is in the map // Check if the reported capability is in the map
if _, exists := capMap[reportedCapability]; exists { if _, exists := capMap[reportedCapability]; exists {
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("Using reported capability") xlog.Debug("Using reported capability", "reportedCapability", reportedCapability, "capMap", capMap)
return reportedCapability return reportedCapability
} }
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("The requested capability was not found, using default capability") xlog.Debug("The requested capability was not found, using default capability", "reportedCapability", reportedCapability, "capMap", capMap)
// Otherwise, return the default capability (catch-all) // Otherwise, return the default capability (catch-all)
return defaultCapability return defaultCapability
} }
@@ -61,7 +61,7 @@ func (s *SystemState) Capability(capMap map[string]string) string {
func (s *SystemState) getSystemCapabilities() string { func (s *SystemState) getSystemCapabilities() string {
capability := os.Getenv(capabilityEnv) capability := os.Getenv(capabilityEnv)
if capability != "" { if capability != "" {
log.Info().Str("capability", capability).Msgf("Using forced capability from environment variable (%s)", capabilityEnv) xlog.Info("Using forced capability from environment variable", "capability", capability, "env", capabilityEnv)
return capability return capability
} }
@@ -77,27 +77,27 @@ func (s *SystemState) getSystemCapabilities() string {
if _, err := os.Stat(capabilityRunFile); err == nil { if _, err := os.Stat(capabilityRunFile); err == nil {
capability, err := os.ReadFile(capabilityRunFile) capability, err := os.ReadFile(capabilityRunFile)
if err == nil { if err == nil {
log.Info().Str("capabilityRunFile", capabilityRunFile).Str("capability", string(capability)).Msgf("Using forced capability run file (%s)", capabilityRunFileEnv) xlog.Info("Using forced capability run file", "capabilityRunFile", capabilityRunFile, "capability", string(capability), "env", capabilityRunFileEnv)
return strings.Trim(strings.TrimSpace(string(capability)), "\n") return strings.Trim(strings.TrimSpace(string(capability)), "\n")
} }
} }
// If we are on mac and arm64, we will return metal // If we are on mac and arm64, we will return metal
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
log.Info().Msgf("Using metal capability (arm64 on mac), set %s to override", capabilityEnv) xlog.Info("Using metal capability (arm64 on mac)", "env", capabilityEnv)
return metal return metal
} }
// If we are on mac and x86, we will return darwin-x86 // If we are on mac and x86, we will return darwin-x86
if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
log.Info().Msgf("Using darwin-x86 capability (amd64 on mac), set %s to override", capabilityEnv) xlog.Info("Using darwin-x86 capability (amd64 on mac)", "env", capabilityEnv)
return darwinX86 return darwinX86
} }
// If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t // If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t
if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" {
if s.GPUVendor == nvidia { if s.GPUVendor == nvidia {
log.Info().Msgf("Using nvidia-l4t capability (arm64 on linux), set %s to override", capabilityEnv) xlog.Info("Using nvidia-l4t capability (arm64 on linux)", "env", capabilityEnv)
if cuda13DirExists { if cuda13DirExists {
return nvidiaL4TCuda13 return nvidiaL4TCuda13
} }
@@ -117,14 +117,14 @@ func (s *SystemState) getSystemCapabilities() string {
} }
if s.GPUVendor == "" { if s.GPUVendor == "" {
log.Info().Msgf("Default capability (no GPU detected), set %s to override", capabilityEnv) xlog.Info("Default capability (no GPU detected)", "env", capabilityEnv)
return defaultCapability return defaultCapability
} }
log.Info().Str("Capability", s.GPUVendor).Msgf("Capability automatically detected, set %s to override", capabilityEnv) xlog.Info("Capability automatically detected", "capability", s.GPUVendor, "env", capabilityEnv)
// If vram is less than 4GB, let's default to CPU but warn the user that they can override that via env // If vram is less than 4GB, let's default to CPU but warn the user that they can override that via env
if s.VRAM <= 4*1024*1024*1024 { if s.VRAM <= 4*1024*1024*1024 {
log.Warn().Msgf("VRAM is less than 4GB, defaulting to CPU. Set %s to override", capabilityEnv) xlog.Warn("VRAM is less than 4GB, defaulting to CPU", "env", capabilityEnv)
return defaultCapability return defaultCapability
} }

8
pkg/system/state.go
View File

@@ -3,7 +3,7 @@ package system
import ( import (
"github.com/jaypipes/ghw/pkg/gpu" "github.com/jaypipes/ghw/pkg/gpu"
"github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
type Backend struct { type Backend struct {
@@ -51,11 +51,11 @@ func GetSystemState(opts ...SystemStateOptions) (*SystemState, error) {
// Detection is best-effort here, we don't want to fail if it fails // Detection is best-effort here, we don't want to fail if it fails
state.gpus, _ = xsysinfo.GPUs() state.gpus, _ = xsysinfo.GPUs()
log.Debug().Any("gpus", state.gpus).Msg("GPUs") xlog.Debug("GPUs", "gpus", state.gpus)
state.GPUVendor, _ = detectGPUVendor(state.gpus) state.GPUVendor, _ = detectGPUVendor(state.gpus)
log.Debug().Str("gpuVendor", state.GPUVendor).Msg("GPU vendor") xlog.Debug("GPU vendor", "gpuVendor", state.GPUVendor)
state.VRAM, _ = xsysinfo.TotalAvailableVRAM() state.VRAM, _ = xsysinfo.TotalAvailableVRAM()
log.Debug().Any("vram", state.VRAM).Msg("Total available VRAM") xlog.Debug("Total available VRAM", "vram", state.VRAM)
return state, nil return state, nil
} }

102
pkg/utils/base64.go
View File

@@ -1,51 +1,51 @@
package utils package utils
import ( import (
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"regexp" "regexp"
"strings" "strings"
"time" "time"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
var base64DownloadClient http.Client = http.Client{ var base64DownloadClient http.Client = http.Client{
Timeout: 30 * time.Second, Timeout: 30 * time.Second,
} }
var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`) var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
// GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers // GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers
func GetContentURIAsBase64(s string) (string, error) { func GetContentURIAsBase64(s string) (string, error) {
if strings.HasPrefix(s, "http") || strings.HasPrefix(s, "https") { if strings.HasPrefix(s, "http") || strings.HasPrefix(s, "https") {
// download the image // download the image
resp, err := base64DownloadClient.Get(s) resp, err := base64DownloadClient.Get(s)
if err != nil { if err != nil {
return "", err return "", err
} }
defer resp.Body.Close() defer resp.Body.Close()
// read the image data into memory // read the image data into memory
data, err := io.ReadAll(resp.Body) data, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", err return "", err
} }
// encode the image data in base64 // encode the image data in base64
encoded := base64.StdEncoding.EncodeToString(data) encoded := base64.StdEncoding.EncodeToString(data)
// return the base64 string // return the base64 string
return encoded, nil return encoded, nil
} }
// Match any data URI prefix pattern // Match any data URI prefix pattern
if match := dataURIPattern.FindString(s); match != "" { if match := dataURIPattern.FindString(s); match != "" {
log.Debug().Msgf("Found data URI prefix: %s", match) xlog.Debug("Found data URI prefix", "prefix", match)
return strings.Replace(s, match, "", 1), nil return strings.Replace(s, match, "", 1), nil
} }
return "", fmt.Errorf("not valid base64 data type string") return "", fmt.Errorf("not valid base64 data type string")
} }

6
pkg/utils/logging.go
View File

@@ -3,7 +3,7 @@ package utils
import ( import (
"time" "time"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
var lastProgress time.Time = time.Now() var lastProgress time.Time = time.Now()
@@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
} }
if total != "" { if total != "" {
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta) xlog.Info("Downloading", "fileName", fileName, "current", current, "total", total, "percentage", percentage, "eta", eta)
} else { } else {
log.Info().Msgf("Downloading: %s", current) xlog.Info("Downloading", "current", current)
} }
} }
} }

29
pkg/xsysinfo/gpu.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/jaypipes/ghw" "github.com/jaypipes/ghw"
"github.com/jaypipes/ghw/pkg/gpu" "github.com/jaypipes/ghw/pkg/gpu"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// GPU vendor constants // GPU vendor constants
@@ -213,7 +213,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr cmd.Stderr = &stderr
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("nvidia-smi failed") xlog.Debug("nvidia-smi failed", "error", err, "stderr", stderr.String())
return nil return nil
} }
@@ -246,7 +246,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
// Unified memory device - fall back to system RAM // Unified memory device - fall back to system RAM
sysInfo, err := GetSystemRAMInfo() sysInfo, err := GetSystemRAMInfo()
if err != nil { if err != nil {
log.Debug().Err(err).Str("device", name).Msg("failed to get system RAM for unified memory device") xlog.Debug("failed to get system RAM for unified memory device", "error", err, "device", name)
// Still add the GPU but with zero memory info // Still add the GPU but with zero memory info
gpus = append(gpus, GPUMemoryInfo{ gpus = append(gpus, GPUMemoryInfo{
Index: idx, Index: idx,
@@ -267,13 +267,10 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
usagePercent = float64(usedBytes) / float64(totalBytes) * 100 usagePercent = float64(usedBytes) / float64(totalBytes) * 100
} }
log.Debug(). xlog.Debug("using system RAM for unified memory GPU", "device", name, "system_ram_bytes", totalBytes)
Str("device", name).
Uint64("system_ram_bytes", totalBytes).
Msg("using system RAM for unified memory GPU")
} else if isNA { } else if isNA {
// Unknown device with N/A values - skip memory info // Unknown device with N/A values - skip memory info
log.Debug().Str("device", name).Msg("nvidia-smi returned N/A for unknown device") xlog.Debug("nvidia-smi returned N/A for unknown device", "device", name)
gpus = append(gpus, GPUMemoryInfo{ gpus = append(gpus, GPUMemoryInfo{
Index: idx, Index: idx,
Name: name, Name: name,
@@ -329,7 +326,7 @@ func getAMDGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr cmd.Stderr = &stderr
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("rocm-smi failed") xlog.Debug("rocm-smi failed", "error", err, "stderr", stderr.String())
return nil return nil
} }
@@ -416,7 +413,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
cmd.Stderr = &stderr cmd.Stderr = &stderr
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("xpu-smi discovery failed") xlog.Debug("xpu-smi discovery failed", "error", err, "stderr", stderr.String())
return nil return nil
} }
@@ -431,7 +428,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
} }
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse xpu-smi discovery output") xlog.Debug("failed to parse xpu-smi discovery output", "error", err)
return nil return nil
} }
@@ -494,7 +491,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
cmd.Stderr = &stderr cmd.Stderr = &stderr
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("intel_gpu_top failed") xlog.Debug("intel_gpu_top failed", "error", err, "stderr", stderr.String())
return nil return nil
} }
@@ -523,7 +520,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
} }
if err := json.Unmarshal([]byte(lastJSON), &result); err != nil { if err := json.Unmarshal([]byte(lastJSON), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse intel_gpu_top output") xlog.Debug("failed to parse intel_gpu_top output", "error", err)
return nil return nil
} }
@@ -557,7 +554,7 @@ func GetResourceInfo() ResourceInfo {
// No GPU - fall back to system RAM // No GPU - fall back to system RAM
ramInfo, err := GetSystemRAMInfo() ramInfo, err := GetSystemRAMInfo()
if err != nil { if err != nil {
log.Debug().Err(err).Msg("failed to get system RAM info") xlog.Debug("failed to get system RAM info", "error", err)
return ResourceInfo{ return ResourceInfo{
Type: "ram", Type: "ram",
Available: false, Available: false,
@@ -601,7 +598,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr cmd.Stderr = &stderr
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("vulkaninfo failed") xlog.Debug("vulkaninfo failed", "error", err, "stderr", stderr.String())
return nil return nil
} }
@@ -620,7 +617,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
} }
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse vulkaninfo output") xlog.Debug("failed to parse vulkaninfo output", "error", err)
return nil return nil
} }

4
pkg/xsysinfo/memory.go
View File

@@ -2,7 +2,7 @@ package xsysinfo
import ( import (
"github.com/mudler/memory" "github.com/mudler/memory"
"github.com/rs/zerolog/log" "github.com/mudler/xlog"
) )
// SystemRAMInfo contains system RAM usage information // SystemRAMInfo contains system RAM usage information
@@ -25,7 +25,7 @@ func GetSystemRAMInfo() (*SystemRAMInfo, error) {
if total > 0 { if total > 0 {
usagePercent = float64(used) / float64(total) * 100 usagePercent = float64(used) / float64(total) * 100
} }
log.Debug().Uint64("total", total).Uint64("used", used).Uint64("free", free).Float64("usage_percent", usagePercent).Msg("System RAM Info") xlog.Debug("System RAM Info", "total", total, "used", used, "free", free, "usage_percent", usagePercent)
return &SystemRAMInfo{ return &SystemRAMInfo{
Total: total, Total: total,
Used: used, Used: used,

5
tests/integration/integration_suite_test.go
View File

@@ -6,12 +6,11 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/rs/zerolog" "github.com/mudler/xlog"
"github.com/rs/zerolog/log"
) )
func TestLocalAI(t *testing.T) { func TestLocalAI(t *testing.T) {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
RegisterFailHandler(Fail) RegisterFailHandler(Fail)
RunSpecs(t, "LocalAI test suite") RunSpecs(t, "LocalAI test suite")
} }

7
tests/integration/stores_test.go
View File

@@ -8,8 +8,7 @@ import (
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/rs/zerolog" "github.com/mudler/xlog"
"github.com/rs/zerolog/log"
"github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/grpc"
@@ -187,7 +186,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
for i, k := range keys { for i, k := range keys {
s := sims[i] s := sims[i]
log.Debug().Float32("similarity", s).Msgf("key: %v", k) xlog.Debug("key", "similarity", s, "key", k)
} }
Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5})) Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5}))
@@ -214,7 +213,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
for i, k := range ks { for i, k := range ks {
s := sims[i] s := sims[i]
log.Debug().Float32("similarity", s).Msgf("key: %v", k) xlog.Debug("key", "similarity", s, "key", k)
} }
Expect(ks[0]).To(Equal(keys[0])) Expect(ks[0]).To(Equal(keys[0]))