mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-04-29 03:24:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore(refactor): move logging to common package based on slog (#7668)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-12-21 19:33:13 +01:00
committed by GitHub
parent 38cde81ff4
commit c37785b78c
88 changed files with 1250 additions and 1320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/go/local-store/debug.go
View File

@@ -4,11 +4,11 @@
package main
import (
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func assert(cond bool, msg string) {
if !cond {
log.Fatal().Stack().Msg(msg)
xlog.Fatal().Stack().Msg(msg)
}
}

6
backend/go/local-store/main.go
View File

@@ -4,11 +4,9 @@ package main
import (
"flag"
"os"
grpc "github.com/mudler/LocalAI/pkg/grpc"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
var (
@@ -16,7 +14,7 @@ var (
)
func main() {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
flag.Parse()

14
backend/go/local-store/store.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/grpc/base"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type Store struct {
@@ -135,7 +135,7 @@ func (s *Store) StoresSet(opts *pb.StoresSetOptions) error {
} else {
sample = k.Floats
}
log.Debug().Msgf("Key is not normalized: %v", sample)
xlog.Debug("Key is not normalized", "sample", sample)
}
kvs[i] = Pair{
@@ -238,7 +238,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k))
}
log.Debug().Msgf("Delete: found = %v, t = %d, j = %d, len(merge_ks) = %d, len(merge_vs) = %d", found, len(tail_ks), j, len(merge_ks), len(merge_vs))
xlog.Debug("Delete", "found", found, "tailLen", len(tail_ks), "j", j, "mergeKeysLen", len(merge_ks), "mergeValuesLen", len(merge_vs))
}
merge_ks = append(merge_ks, tail_ks...)
@@ -261,7 +261,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
}(), "Keys to delete still present")
if len(s.keys) != l {
log.Debug().Msgf("Delete: Some keys not found: len(s.keys) = %d, l = %d", len(s.keys), l)
xlog.Debug("Delete: Some keys not found", "keysLen", len(s.keys), "expectedLen", l)
}
return nil
@@ -273,7 +273,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
ks := sortIntoKeySlicese(opts.Keys)
if len(s.keys) == 0 {
log.Debug().Msgf("Get: No keys in store")
xlog.Debug("Get: No keys in store")
}
if s.keyLen == -1 {
@@ -305,7 +305,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
}
if len(pbKeys) != len(opts.Keys) {
log.Debug().Msgf("Get: Some keys not found: len(pbKeys) = %d, len(opts.Keys) = %d, len(s.Keys) = %d", len(pbKeys), len(opts.Keys), len(s.keys))
xlog.Debug("Get: Some keys not found", "pbKeysLen", len(pbKeys), "optsKeysLen", len(opts.Keys), "storeKeysLen", len(s.keys))
}
return pb.StoresGetResult{
@@ -507,7 +507,7 @@ func (s *Store) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, err
} else {
sample = tk
}
log.Debug().Msgf("Trying to compare non-normalized key with normalized keys: %v", sample)
xlog.Debug("Trying to compare non-normalized key with normalized keys", "sample", sample)
}
return s.StoresFindFallback(opts)

40
cmd/local-ai/main.go
View File

@@ -8,10 +8,7 @@ import (
"github.com/joho/godotenv"
"github.com/mudler/LocalAI/core/cli"
"github.com/mudler/LocalAI/internal"
"github.com/mudler/cogito/pkg/xlog"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
_ "github.com/mudler/LocalAI/swagger"
)
@@ -19,9 +16,8 @@ import (
func main() {
var err error
// Initialize zerolog at a level of INFO, we will set the desired level after we parse the CLI options
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
zerolog.SetGlobalLevel(zerolog.InfoLevel)
// Initialize xlog at a level of INFO, we will set the desired level after we parse the CLI options
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
// handle loading environment variables from .env files
envFiles := []string{".env", "localai.env"}
@@ -33,10 +29,10 @@ func main() {
for _, envFile := range envFiles {
if _, err := os.Stat(envFile); err == nil {
log.Debug().Str("envFile", envFile).Msg("env file found, loading environment variables from file")
xlog.Debug("env file found, loading environment variables from file", "envFile", envFile)
err = godotenv.Load(envFile)
if err != nil {
log.Error().Err(err).Str("envFile", envFile).Msg("failed to load environment variables from file")
xlog.Error("failed to load environment variables from file", "error", err, "envFile", envFile)
continue
}
}
@@ -68,7 +64,6 @@ Version: ${version}
logLevel := "info"
if cli.CLI.Debug && cli.CLI.LogLevel == nil {
logLevel = "debug"
zerolog.SetGlobalLevel(zerolog.DebugLevel)
cli.CLI.LogLevel = &logLevel
}
@@ -76,31 +71,12 @@ Version: ${version}
cli.CLI.LogLevel = &logLevel
}
// Set cogito logger to the same level as our logger
// Leave an empty format type
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), ""))
switch *cli.CLI.LogLevel {
case "error":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
log.Debug().Msg("Setting logging to error")
case "warn":
zerolog.SetGlobalLevel(zerolog.WarnLevel)
log.Debug().Msg("Setting logging to warn")
case "info":
zerolog.SetGlobalLevel(zerolog.InfoLevel)
log.Debug().Msg("Setting logging to info")
case "debug":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
log.Debug().Msg("Setting logging to debug")
case "trace":
zerolog.SetGlobalLevel(zerolog.TraceLevel)
log.Debug().Msg("Setting logging to trace")
}
// Set xlog logger with the desired level and text format
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), "text"))
// Run the thing!
err = ctx.Run(&cli.CLI.Context)
if err != nil {
log.Fatal().Err(err).Msg("Error running the application")
xlog.Fatal("Error running the application", "error", err)
}
}

9
core/application/agent_jobs.go
View File

@@ -4,7 +4,7 @@ import (
"time"
"github.com/mudler/LocalAI/core/services"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// RestartAgentJobService restarts the agent job service with current ApplicationConfig settings
@@ -15,7 +15,7 @@ func (a *Application) RestartAgentJobService() error {
// Stop existing service if running
if a.agentJobService != nil {
if err := a.agentJobService.Stop(); err != nil {
log.Warn().Err(err).Msg("Error stopping agent job service")
xlog.Warn("Error stopping agent job service", "error", err)
}
// Wait a bit for shutdown to complete
time.Sleep(200 * time.Millisecond)
@@ -32,12 +32,11 @@ func (a *Application) RestartAgentJobService() error {
// Start the service
err := agentJobService.Start(a.ApplicationConfig().Context)
if err != nil {
log.Error().Err(err).Msg("Failed to start agent job service")
xlog.Error("Failed to start agent job service", "error", err)
return err
}
a.agentJobService = agentJobService
log.Info().Msg("Agent job service restarted")
xlog.Info("Agent job service restarted")
return nil
}

41
core/application/config_file_watcher.go
View File

@@ -11,7 +11,7 @@ import (
"dario.cat/mergo"
"github.com/fsnotify/fsnotify"
"github.com/mudler/LocalAI/core/config"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
@@ -33,15 +33,15 @@ func newConfigFileHandler(appConfig *config.ApplicationConfig) configFileHandler
}
err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
if err != nil {
log.Error().Err(err).Str("file", "api_keys.json").Msg("unable to register config file handler")
xlog.Error("unable to register config file handler", "error", err, "file", "api_keys.json")
}
err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
if err != nil {
log.Error().Err(err).Str("file", "external_backends.json").Msg("unable to register config file handler")
xlog.Error("unable to register config file handler", "error", err, "file", "external_backends.json")
}
err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true)
if err != nil {
log.Error().Err(err).Str("file", "runtime_settings.json").Msg("unable to register config file handler")
xlog.Error("unable to register config file handler", "error", err, "file", "runtime_settings.json")
}
// Note: agent_tasks.json and agent_jobs.json are handled by AgentJobService directly
// The service watches and reloads these files internally
@@ -62,14 +62,14 @@ func (c *configFileHandler) Register(filename string, handler fileHandler, runNo
func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
log.Trace().Str("filename", rootedFilePath).Msg("reading file for dynamic config update")
xlog.Debug("reading file for dynamic config update", "filename", rootedFilePath)
fileContent, err := os.ReadFile(rootedFilePath)
if err != nil && !os.IsNotExist(err) {
log.Error().Err(err).Str("filename", rootedFilePath).Msg("could not read file")
xlog.Error("could not read file", "error", err, "filename", rootedFilePath)
}
if err = handler(fileContent, c.appConfig); err != nil {
log.Error().Err(err).Msg("WatchConfigDirectory goroutine failed to update options")
xlog.Error("WatchConfigDirectory goroutine failed to update options", "error", err)
}
}
@@ -81,13 +81,13 @@ func (c *configFileHandler) Watch() error {
}
if c.appConfig.DynamicConfigsDirPollInterval > 0 {
log.Debug().Msg("Poll interval set, falling back to polling for configuration changes")
xlog.Debug("Poll interval set, falling back to polling for configuration changes")
ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
go func() {
for {
<-ticker.C
for file, handler := range c.handlers {
log.Debug().Str("file", file).Msg("polling config file")
xlog.Debug("polling config file", "file", file)
c.callHandler(file, handler)
}
}
@@ -111,7 +111,7 @@ func (c *configFileHandler) Watch() error {
c.callHandler(filepath.Base(event.Name), handler)
}
case err, ok := <-c.watcher.Errors:
log.Error().Err(err).Msg("config watcher error received")
xlog.Error("config watcher error received", "error", err)
if !ok {
return
}
@@ -135,8 +135,7 @@ func (c *configFileHandler) Stop() error {
func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing api keys runtime update")
log.Trace().Int("numKeys", len(startupAppConfig.ApiKeys)).Msg("api keys provided at startup")
xlog.Debug("processing api keys runtime update", "numKeys", len(startupAppConfig.ApiKeys))
if len(fileContent) > 0 {
// Parse JSON content from the file
@@ -146,14 +145,14 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
return err
}
log.Trace().Int("numKeys", len(fileKeys)).Msg("discovered API keys from api keys dynamic config dile")
xlog.Debug("discovered API keys from api keys dynamic config file", "numKeys", len(fileKeys))
appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
} else {
log.Trace().Msg("no API keys discovered from dynamic config file")
xlog.Debug("no API keys discovered from dynamic config file")
appConfig.ApiKeys = startupAppConfig.ApiKeys
}
log.Trace().Int("numKeys", len(appConfig.ApiKeys)).Msg("total api keys after processing")
xlog.Debug("total api keys after processing", "numKeys", len(appConfig.ApiKeys))
return nil
}
@@ -162,7 +161,7 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing external_backends.json")
xlog.Debug("processing external_backends.json")
if len(fileContent) > 0 {
// Parse JSON content from the file
@@ -179,7 +178,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
} else {
appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
}
log.Debug().Msg("external backends loaded from external_backends.json")
xlog.Debug("external backends loaded from external_backends.json")
return nil
}
return handler
@@ -187,7 +186,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler {
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
log.Debug().Msg("processing runtime_settings.json")
xlog.Debug("processing runtime_settings.json")
// Determine if settings came from env vars by comparing with startup config
// startupAppConfig contains the original values set from env vars at startup.
@@ -241,7 +240,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
if err == nil {
appConfig.WatchDogIdleTimeout = dur
} else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
}
}
if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout {
@@ -249,7 +248,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
if err == nil {
appConfig.WatchDogBusyTimeout = dur
} else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
}
}
// Handle MaxActiveBackends (new) and SingleBackend (deprecated)
@@ -340,7 +339,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
}
}
}
log.Debug().Msg("runtime settings loaded from runtime_settings.json")
xlog.Debug("runtime settings loaded from runtime_settings.json")
return nil
}
return handler

31
core/application/p2p.go
View File

@@ -14,8 +14,7 @@ import (
"github.com/mudler/LocalAI/core/services"
"github.com/mudler/edgevpn/pkg/node"
"github.com/rs/zerolog/log"
zlog "github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func (a *Application) StopP2P() error {
@@ -86,14 +85,14 @@ func (a *Application) StartP2P() error {
}
// Attach a ServiceDiscoverer to the p2p node
log.Info().Msg("Starting P2P server discovery...")
xlog.Info("Starting P2P server discovery...")
if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
var tunnelAddresses []string
for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
if v.IsOnline() {
tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
} else {
log.Info().Msgf("Node %s is offline", v.ID)
xlog.Info("Node is offline", "node", v.ID)
}
}
if a.applicationConfig.TunnelCallback != nil {
@@ -143,17 +142,17 @@ func (a *Application) RestartP2P() error {
// Start P2P stack in a goroutine
go func() {
if err := a.StartP2P(); err != nil {
log.Error().Err(err).Msg("Failed to start P2P stack")
xlog.Error("Failed to start P2P stack", "error", err)
cancel() // Cancel context on error
}
}()
log.Info().Msg("P2P stack restarted with new settings")
xlog.Info("P2P stack restarted with new settings")
return nil
}
func syncState(ctx context.Context, n *node.Node, app *Application) error {
zlog.Debug().Msg("[p2p-sync] Syncing state")
xlog.Debug("[p2p-sync] Syncing state")
whatWeHave := []string{}
for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
@@ -162,20 +161,20 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
ledger, _ := n.Ledger()
currentData := ledger.CurrentData()
zlog.Debug().Msgf("[p2p-sync] Current data: %v", currentData)
xlog.Debug("[p2p-sync] Current data", "data", currentData)
data, exists := ledger.GetKey("shared_state", "models")
if !exists {
ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
zlog.Debug().Msgf("No models found in the ledger, announced our models: %v", whatWeHave)
xlog.Debug("No models found in the ledger, announced our models", "models", whatWeHave)
}
models := []string{}
if err := data.Unmarshal(&models); err != nil {
zlog.Warn().Err(err).Msg("error unmarshalling models")
xlog.Warn("error unmarshalling models", "error", err)
return nil
}
zlog.Debug().Msgf("[p2p-sync] Models that are present in this instance: %v\nModels that are in the ledger: %v", whatWeHave, models)
xlog.Debug("[p2p-sync] Models comparison", "ourModels", whatWeHave, "ledgerModels", models)
// Sync with our state
whatIsNotThere := []string{}
@@ -185,7 +184,7 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
}
}
if len(whatIsNotThere) > 0 {
zlog.Debug().Msgf("[p2p-sync] Announcing our models: %v", append(models, whatIsNotThere...))
xlog.Debug("[p2p-sync] Announcing our models", "models", append(models, whatIsNotThere...))
ledger.AnnounceUpdate(
ctx,
1*time.Minute,
@@ -198,16 +197,16 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
// Check if we have a model that is not in our state, otherwise install it
for _, model := range models {
if slices.Contains(whatWeHave, model) {
zlog.Debug().Msgf("[p2p-sync] Model %s is already present in this instance", model)
xlog.Debug("[p2p-sync] Model is already present in this instance", "model", model)
continue
}
// we install model
zlog.Info().Msgf("[p2p-sync] Installing model which is not present in this instance: %s", model)
xlog.Info("[p2p-sync] Installing model which is not present in this instance", "model", model)
uuid, err := uuid.NewUUID()
if err != nil {
zlog.Error().Err(err).Msg("error generating UUID")
xlog.Error("error generating UUID", "error", err)
continue
}
@@ -230,7 +229,7 @@ func (a *Application) p2pSync(ctx context.Context, n *node.Node) error {
return
case <-time.After(1 * time.Minute):
if err := syncState(ctx, n, a); err != nil {
zlog.Error().Err(err).Msg("error syncing state")
xlog.Error("error syncing state", "error", err)
}
}

56
core/application/startup.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func New(opts ...config.AppOption) (*Application, error) {
@@ -28,8 +28,8 @@ func New(opts ...config.AppOption) (*Application, error) {
application := newApplication(options)
application.startupConfig = &startupConfigCopy
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.SystemState.Model.ModelsPath)
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
xlog.Info("Starting LocalAI", "threads", options.Threads, "modelsPath", options.SystemState.Model.ModelsPath)
xlog.Info("LocalAI version", "version", internal.PrintableVersion())
if err := application.start(); err != nil {
return nil, err
@@ -37,14 +37,14 @@ func New(opts ...config.AppOption) (*Application, error) {
caps, err := xsysinfo.CPUCapabilities()
if err == nil {
log.Debug().Msgf("CPU capabilities: %v", caps)
xlog.Debug("CPU capabilities", "capabilities", caps)
}
gpus, err := xsysinfo.GPUs()
if err == nil {
log.Debug().Msgf("GPU count: %d", len(gpus))
xlog.Debug("GPU count", "count", len(gpus))
for _, gpu := range gpus {
log.Debug().Msgf("GPU: %s", gpu.String())
xlog.Debug("GPU", "gpu", gpu.String())
}
}
@@ -71,33 +71,33 @@ func New(opts ...config.AppOption) (*Application, error) {
}
if err := coreStartup.InstallModels(options.Context, application.GalleryService(), options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
log.Error().Err(err).Msg("error installing models")
xlog.Error("error installing models", "error", err)
}
for _, backend := range options.ExternalBackends {
if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
log.Error().Err(err).Msg("error installing external backend")
xlog.Error("error installing external backend", "error", err)
}
}
configLoaderOpts := options.ToConfigLoaderOptions()
if err := application.ModelConfigLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil {
log.Error().Err(err).Msg("error loading config files")
xlog.Error("error loading config files", "error", err)
}
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
log.Error().Err(err).Msg("error registering external backends")
xlog.Error("error registering external backends", "error", err)
}
if options.ConfigFile != "" {
if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
log.Error().Err(err).Msg("error loading config file")
xlog.Error("error loading config file", "error", err)
}
}
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
log.Error().Err(err).Msg("error downloading models")
xlog.Error("error downloading models", "error", err)
}
if options.PreloadJSONModels != "" {
@@ -114,7 +114,7 @@ func New(opts ...config.AppOption) (*Application, error) {
if options.Debug {
for _, v := range application.ModelConfigLoader().GetAllModelsConfigs() {
log.Debug().Msgf("Model: %s (config: %+v)", v.Name, v)
xlog.Debug("Model", "name", v.Name, "config", v)
}
}
@@ -128,10 +128,10 @@ func New(opts ...config.AppOption) (*Application, error) {
// turn off any process that was started by GRPC if the context is canceled
go func() {
<-options.Context.Done()
log.Debug().Msgf("Context canceled, shutting down")
xlog.Debug("Context canceled, shutting down")
err := application.ModelLoader().StopAllGRPC()
if err != nil {
log.Error().Err(err).Msg("error while stopping all grpc backends")
xlog.Error("error while stopping all grpc backends", "error", err)
}
}()
@@ -145,7 +145,7 @@ func New(opts ...config.AppOption) (*Application, error) {
return nil, err
}
log.Debug().Msgf("Auto loading model %s into memory from file: %s", m, cfg.Model)
xlog.Debug("Auto loading model into memory from file", "model", m, "file", cfg.Model)
o := backend.ModelOptions(*cfg, options)
@@ -160,7 +160,7 @@ func New(opts ...config.AppOption) (*Application, error) {
// Watch the configuration directory
startWatcher(options)
log.Info().Msg("core/startup process completed!")
xlog.Info("core/startup process completed!")
return application, nil
}
@@ -174,18 +174,18 @@ func startWatcher(options *config.ApplicationConfig) {
if os.IsNotExist(err) {
// We try to create the directory if it does not exist and was specified
if err := os.MkdirAll(options.DynamicConfigsDir, 0700); err != nil {
log.Error().Err(err).Msg("failed creating DynamicConfigsDir")
xlog.Error("failed creating DynamicConfigsDir", "error", err)
}
} else {
// something else happened, we log the error and don't start the watcher
log.Error().Err(err).Msg("failed to read DynamicConfigsDir, watcher will not be started")
xlog.Error("failed to read DynamicConfigsDir, watcher will not be started", "error", err)
return
}
}
configHandler := newConfigFileHandler(options)
if err := configHandler.Watch(); err != nil {
log.Error().Err(err).Msg("failed creating watcher")
xlog.Error("failed creating watcher", "error", err)
}
}
@@ -211,17 +211,17 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
fileContent, err := os.ReadFile(settingsFile)
if err != nil {
if os.IsNotExist(err) {
log.Debug().Msg("runtime_settings.json not found, using defaults")
xlog.Debug("runtime_settings.json not found, using defaults")
return
}
log.Warn().Err(err).Msg("failed to read runtime_settings.json")
xlog.Warn("failed to read runtime_settings.json", "error", err)
return
}
var settings config.RuntimeSettings
if err := json.Unmarshal(fileContent, &settings); err != nil {
log.Warn().Err(err).Msg("failed to parse runtime_settings.json")
xlog.Warn("failed to parse runtime_settings.json", "error", err)
return
}
@@ -257,7 +257,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil {
options.WatchDogIdleTimeout = dur
} else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
}
}
}
@@ -267,7 +267,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil {
options.WatchDogBusyTimeout = dur
} else {
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
}
}
}
@@ -277,7 +277,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
if err == nil {
options.WatchDogInterval = dur
} else {
log.Warn().Err(err).Str("interval", *settings.WatchdogInterval).Msg("invalid watchdog interval in runtime_settings.json")
xlog.Warn("invalid watchdog interval in runtime_settings.json", "error", err, "interval", *settings.WatchdogInterval)
options.WatchDogInterval = model.DefaultWatchdogInterval
}
}
@@ -331,7 +331,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
}
}
log.Debug().Msg("Runtime settings loaded from runtime_settings.json")
xlog.Debug("Runtime settings loaded from runtime_settings.json")
}
// initializeWatchdog initializes the watchdog with current ApplicationConfig settings
@@ -362,7 +362,7 @@ func initializeWatchdog(application *Application, options *config.ApplicationCon
go func() {
<-options.Context.Done()
log.Debug().Msgf("Context canceled, shutting down")
xlog.Debug("Context canceled, shutting down")
wd.Shutdown()
}()
}

17
core/application/watchdog.go
View File

@@ -4,7 +4,7 @@ import (
"time"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func (a *Application) StopWatchdog() error {
@@ -52,24 +52,17 @@ func (a *Application) startWatchdog() error {
go func() {
select {
case <-a.watchdogStop:
log.Debug().Msg("Watchdog stop signal received")
xlog.Debug("Watchdog stop signal received")
wd.Shutdown()
case <-appConfig.Context.Done():
log.Debug().Msg("Context canceled, shutting down watchdog")
xlog.Debug("Context canceled, shutting down watchdog")
wd.Shutdown()
}
}()
log.Info().
Int("lruLimit", lruLimit).
Bool("busyCheck", appConfig.WatchDogBusy).
Bool("idleCheck", appConfig.WatchDogIdle).
Bool("memoryReclaimer", appConfig.MemoryReclaimerEnabled).
Float64("memoryThreshold", appConfig.MemoryReclaimerThreshold).
Dur("interval", appConfig.WatchDogInterval).
Msg("Watchdog started with new settings")
xlog.Info("Watchdog started with new settings", "lruLimit", lruLimit, "busyCheck", appConfig.WatchDogBusy, "idleCheck", appConfig.WatchDogIdle, "memoryReclaimer", appConfig.MemoryReclaimerEnabled, "memoryThreshold", appConfig.MemoryReclaimerThreshold, "interval", appConfig.WatchDogInterval)
} else {
log.Info().Msg("Watchdog disabled")
xlog.Info("Watchdog disabled")
}
return nil

8
core/backend/llm.go
View File

@@ -9,7 +9,7 @@ import (
"sync"
"unicode/utf8"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema"
@@ -49,7 +49,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
// if we failed to load the model, we try to download it
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
if err != nil {
log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile)
xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
//return nil, err
}
}
@@ -225,7 +225,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
if !ok {
r, err := regexp.Compile(c)
if err != nil {
log.Fatal().Err(err).Msg("failed to compile regex")
xlog.Fatal("failed to compile regex", "error", err)
}
cutstrings[c] = r
reg = cutstrings[c]
@@ -242,7 +242,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
if !ok {
regex, err := regexp.Compile(r)
if err != nil {
log.Fatal().Err(err).Msg("failed to compile regex")
xlog.Fatal("failed to compile regex", "error", err)
}
cutstrings[r] = regex
reg = regex

4
core/backend/options.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/config"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
@@ -208,7 +208,7 @@ func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions
if err == nil {
promptCachePath = p
} else {
log.Error().Err(err).Str("promptCachePath", promptCachePath).Msg("error creating prompt cache folder")
xlog.Error("error creating prompt cache folder", "error", err, "promptCachePath", promptCachePath)
}
}

10
core/cli/backends.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/schollz/progressbar/v3"
)
@@ -49,7 +49,7 @@ type BackendsCMD struct {
func (bl *BackendsList) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery
if err := json.Unmarshal([]byte(bl.BackendGalleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries")
xlog.Error("unable to load galleries", "error", err)
}
systemState, err := system.GetSystemState(
@@ -77,7 +77,7 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error {
func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery
if err := json.Unmarshal([]byte(bi.BackendGalleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries")
xlog.Error("unable to load galleries", "error", err)
}
systemState, err := system.GetSystemState(
@@ -98,7 +98,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
v := int(percentage * 10)
err := progressBar.Set(v)
if err != nil {
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar")
xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
}
}
@@ -113,7 +113,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
for _, backendName := range bu.BackendArgs {
log.Info().Str("backend", backendName).Msg("uninstalling backend")
xlog.Info("uninstalling backend", "backend", backendName)
systemState, err := system.GetSystemState(
system.WithBackendSystemPath(bu.BackendsSystemPath),

4
core/cli/explorer.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/explorer"
"github.com/mudler/LocalAI/core/http"
"github.com/mudler/LocalAI/pkg/signals"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type ExplorerCMD struct {
@@ -51,7 +51,7 @@ func (e *ExplorerCMD) Run(ctx *cliContext.Context) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := appHTTP.Shutdown(ctx); err != nil {
log.Error().Err(err).Msg("error during shutdown")
xlog.Error("error during shutdown", "error", err)
}
})

14
core/cli/models.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/schollz/progressbar/v3"
)
@@ -46,7 +46,7 @@ type ModelsCMD struct {
func (ml *ModelsList) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery
if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries")
xlog.Error("unable to load galleries", "error", err)
}
systemState, err := system.GetSystemState(
@@ -88,12 +88,12 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
var galleries []config.Gallery
if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries")
xlog.Error("unable to load galleries", "error", err)
}
var backendGalleries []config.Gallery
if err := json.Unmarshal([]byte(mi.BackendGalleries), &backendGalleries); err != nil {
log.Error().Err(err).Msg("unable to load backend galleries")
xlog.Error("unable to load backend galleries", "error", err)
}
for _, modelName := range mi.ModelArgs {
@@ -108,7 +108,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
v := int(percentage * 10)
err := progressBar.Set(v)
if err != nil {
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar")
xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
}
}
//startup.InstallModels()
@@ -122,7 +122,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
if !modelURI.LooksLikeOCI() {
model := gallery.FindGalleryElement(models, modelName)
if model == nil {
log.Error().Str("model", modelName).Msg("model not found")
xlog.Error("model not found", "model", modelName)
return err
}
@@ -131,7 +131,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
return err
}
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model")
xlog.Info("installing model", "model", modelName, "license", model.License)
}
modelLoader := model.NewModelLoader(systemState)

19
core/cli/run.go
View File

@@ -15,8 +15,7 @@ import (
"github.com/mudler/LocalAI/internal"
"github.com/mudler/LocalAI/pkg/signals"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type RunCMD struct {
@@ -108,7 +107,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
config.WithYAMLConfigPreload(r.PreloadModelsConfig),
config.WithSystemState(systemState),
config.WithContextSize(r.ContextSize),
config.WithDebug(zerolog.GlobalLevel() <= zerolog.DebugLevel),
config.WithDebug(ctx.Debug || (ctx.LogLevel != nil && *ctx.LogLevel == "debug")),
config.WithGeneratedContentDir(r.GeneratedContentPath),
config.WithUploadDir(r.UploadPath),
config.WithDynamicConfigDir(r.LocalaiConfigDir),
@@ -138,7 +137,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
tunnelEnvVar := strings.Join(tunnels, ",")
// TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable
os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar)
xlog.Debug("setting LLAMACPP_GRPC_SERVERS", "value", tunnelEnvVar)
}),
}
@@ -152,17 +151,17 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
token := ""
if r.Peer2Peer || r.Peer2PeerToken != "" {
log.Info().Msg("P2P mode enabled")
xlog.Info("P2P mode enabled")
token = r.Peer2PeerToken
if token == "" {
// IF no token is provided, and p2p is enabled,
// we generate one and wait for the user to pick up the token (this is for interactive)
log.Info().Msg("No token provided, generating one")
xlog.Info("No token provided, generating one")
token = p2p.GenerateToken(r.Peer2PeerDHTInterval, r.Peer2PeerOTPInterval)
log.Info().Msg("Generated Token:")
xlog.Info("Generated Token:")
fmt.Println(token)
log.Info().Msg("To use the token, you can run the following command in another node or terminal:")
xlog.Info("To use the token, you can run the following command in another node or terminal:")
fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token)
}
opts = append(opts, config.WithP2PToken(token))
@@ -248,7 +247,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
appHTTP, err := http.API(app)
if err != nil {
log.Error().Err(err).Msg("error during HTTP App construction")
xlog.Error("error during HTTP App construction", "error", err)
return err
}
@@ -260,7 +259,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
signals.RegisterGracefulTerminationHandler(func() {
if err := app.ModelLoader().StopAllGRPC(); err != nil {
log.Error().Err(err).Msg("error while stopping all grpc backends")
xlog.Error("error while stopping all grpc backends", "error", err)
}
})

4
core/cli/soundgeneration.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type SoundGenerationCMD struct {
@@ -84,7 +84,7 @@ func (t *SoundGenerationCMD) Run(ctx *cliContext.Context) error {
defer func() {
err := ml.StopAllGRPC()
if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes")
xlog.Error("unable to stop all grpc processes", "error", err)
}
}()

4
core/cli/transcript.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type TranscriptCMD struct {
@@ -54,7 +54,7 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
defer func() {
err := ml.StopAllGRPC()
if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes")
xlog.Error("unable to stop all grpc processes", "error", err)
}
}()

4
core/cli/tts.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type TTSCMD struct {
@@ -53,7 +53,7 @@ func (t *TTSCMD) Run(ctx *cliContext.Context) error {
defer func() {
err := ml.StopAllGRPC()
if err != nil {
log.Error().Err(err).Msg("unable to stop all grpc processes")
xlog.Error("unable to stop all grpc processes", "error", err)
}
}()

50
core/cli/util.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
"github.com/mholt/archiver/v3"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
gguf "github.com/gpustack/gguf-parser-go"
cliContext "github.com/mudler/LocalAI/core/cli/context"
@@ -51,7 +51,7 @@ type CreateOCIImageCMD struct {
}
func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
log.Info().Msg("Creating OCI image from input")
xlog.Info("Creating OCI image from input")
dir, err := os.MkdirTemp("", "localai")
if err != nil {
@@ -62,7 +62,7 @@ func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
if err != nil {
return err
}
log.Info().Msgf("Creating '%s' as '%s' from %v", u.Output, u.Input, u.Input)
xlog.Info("Creating OCI image", "output", u.Output, "input", u.Input)
platform := strings.Split(u.Platform, "/")
if len(platform) != 2 {
@@ -80,27 +80,23 @@ func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error {
f, err := gguf.ParseGGUFFile(u.Args[0])
if err != nil {
// Only valid for gguf files
log.Error().Msgf("guessDefaultsFromFile: %s", "not a GGUF file")
xlog.Error("guessDefaultsFromFile: not a GGUF file")
return err
}
log.Info().
Any("eosTokenID", f.Tokenizer().EOSTokenID).
Any("bosTokenID", f.Tokenizer().BOSTokenID).
Any("modelName", f.Metadata().Name).
Any("architecture", f.Architecture().Architecture).Msgf("GGUF file loaded: %s", u.Args[0])
xlog.Info("GGUF file loaded", "file", u.Args[0], "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
log.Info().Any("tokenizer", fmt.Sprintf("%+v", f.Tokenizer())).Msg("Tokenizer")
log.Info().Any("architecture", fmt.Sprintf("%+v", f.Architecture())).Msg("Architecture")
xlog.Info("Tokenizer", "tokenizer", fmt.Sprintf("%+v", f.Tokenizer()))
xlog.Info("Architecture", "architecture", fmt.Sprintf("%+v", f.Architecture()))
v, exists := f.Header.MetadataKV.Get("tokenizer.chat_template")
if exists {
log.Info().Msgf("chat_template: %s", v.ValueString())
xlog.Info("chat_template", "template", v.ValueString())
}
if u.Header {
for _, metadata := range f.Header.MetadataKV {
log.Info().Msgf("%s: %+v", metadata.Key, metadata.Value)
xlog.Info("metadata", "key", metadata.Key, "value", metadata.Value)
}
// log.Info().Any("header", fmt.Sprintf("%+v", f.Header)).Msg("Header")
}
@@ -117,63 +113,63 @@ func (hfscmd *HFScanCMD) Run(ctx *cliContext.Context) error {
return err
}
log.Info().Msg("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!")
xlog.Info("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!")
if len(hfscmd.ToScan) == 0 {
log.Info().Msg("Checking all installed models against galleries")
xlog.Info("Checking all installed models against galleries")
var galleries []config.Gallery
if err := json.Unmarshal([]byte(hfscmd.Galleries), &galleries); err != nil {
log.Error().Err(err).Msg("unable to load galleries")
xlog.Error("unable to load galleries", "error", err)
}
err := gallery.SafetyScanGalleryModels(galleries, systemState)
if err == nil {
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
} else {
log.Error().Err(err).Msg("! WARNING ! A known-vulnerable model is installed!")
xlog.Error("! WARNING ! A known-vulnerable model is installed!", "error", err)
}
return err
} else {
var errs error = nil
for _, uri := range hfscmd.ToScan {
log.Info().Str("uri", uri).Msg("scanning specific uri")
xlog.Info("scanning specific uri", "uri", uri)
scanResults, err := downloader.HuggingFaceScan(downloader.URI(uri))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Err(err).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("! WARNING ! A known-vulnerable model is included in this repo!")
xlog.Error("! WARNING ! A known-vulnerable model is included in this repo!", "error", err, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
errs = errors.Join(errs, err)
}
}
if errs != nil {
return errs
}
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
return nil
}
}
func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error {
if len(uhcmd.ConfigName) == 0 {
log.Error().Msg("ConfigName is a required parameter")
xlog.Error("ConfigName is a required parameter")
return fmt.Errorf("config name is a required parameter")
}
if len(uhcmd.ModelsPath) == 0 {
log.Error().Msg("ModelsPath is a required parameter")
xlog.Error("ModelsPath is a required parameter")
return fmt.Errorf("model path is a required parameter")
}
bcl := config.NewModelConfigLoader(uhcmd.ModelsPath)
err := bcl.ReadModelConfig(uhcmd.ConfigName)
if err != nil {
log.Error().Err(err).Str("ConfigName", uhcmd.ConfigName).Msg("error while loading backend")
xlog.Error("error while loading backend", "error", err, "ConfigName", uhcmd.ConfigName)
return err
}
bc, exists := bcl.GetModelConfig(uhcmd.ConfigName)
if !exists {
log.Error().Str("ConfigName", uhcmd.ConfigName).Msg("ConfigName not found")
xlog.Error("ConfigName not found", "ConfigName", uhcmd.ConfigName)
}
for name, uc := range config.GetAllModelConfigUsecases() {
if bc.HasUsecases(uc) {
log.Info().Str("Usecase", name)
xlog.Info("Usecase", "usecase", name)
}
}
log.Info().Msg("---")
xlog.Info("---")
return nil
}

10
core/cli/worker/worker_llamacpp.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type LLamaCPP struct {
@@ -30,22 +30,22 @@ const (
func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) {
backends, err := gallery.ListSystemBackends(systemState)
if err != nil {
log.Warn().Msgf("Failed listing system backends: %s", err)
xlog.Warn("Failed listing system backends", "error", err)
return "", err
}
log.Debug().Msgf("System backends: %v", backends)
xlog.Debug("System backends", "backends", backends)
backend, ok := backends.Get(llamaCPPGalleryName)
if !ok {
ml := model.NewModelLoader(systemState)
var gals []config.Gallery
if err := json.Unmarshal([]byte(galleries), &gals); err != nil {
log.Error().Err(err).Msg("failed loading galleries")
xlog.Error("failed loading galleries", "error", err)
return "", err
}
err := gallery.InstallBackendFromGallery(context.Background(), gals, systemState, ml, llamaCPPGalleryName, nil, true)
if err != nil {
log.Error().Err(err).Msg("llama-cpp backend not found, failed to install it")
xlog.Error("llama-cpp backend not found, failed to install it", "error", err)
return "", err
}
}

12
core/cli/worker/worker_p2p.go
View File

@@ -12,8 +12,8 @@ import (
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/pkg/signals"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
"github.com/phayes/freeport"
"github.com/rs/zerolog/log"
)
type P2P struct {
@@ -66,16 +66,16 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
if err != nil {
return err
}
log.Info().Msgf("You need to start llama-cpp-rpc-server on '%s:%s'", address, p)
xlog.Info("You need to start llama-cpp-rpc-server", "address", address, "port", p)
} else {
// Start llama.cpp directly from the version we have pre-packaged
go func() {
for {
log.Info().Msgf("Starting llama-cpp-rpc-server on '%s:%d'", address, port)
xlog.Info("Starting llama-cpp-rpc-server", "address", address, "port", port)
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
if err != nil {
log.Error().Err(err).Msg("Failed to find llama-cpp-rpc-server")
xlog.Error("Failed to find llama-cpp-rpc-server", "error", err)
return
}
@@ -85,7 +85,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ")
}
args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...)
log.Debug().Msgf("Starting llama-cpp-rpc-server on '%s:%d' with args: %+v (%d)", address, port, args, len(args))
xlog.Debug("Starting llama-cpp-rpc-server", "address", address, "port", port, "args", args, "argCount", len(args))
cmd := exec.Command(
grpcProcess, args...,
@@ -97,7 +97,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
cmd.Stdout = os.Stdout
if err := cmd.Start(); err != nil {
log.Error().Any("grpcProcess", grpcProcess).Any("args", args).Err(err).Msg("Failed to start llama-cpp-rpc-server")
xlog.Error("Failed to start llama-cpp-rpc-server", "error", err, "grpcProcess", grpcProcess, "args", args)
}
cmd.Wait()

8
core/config/application_config.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type ApplicationConfig struct {
@@ -298,7 +298,7 @@ func WithStringGalleries(galls string) AppOption {
}
var galleries []Gallery
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
log.Error().Err(err).Msg("failed loading galleries")
xlog.Error("failed loading galleries", "error", err)
}
o.Galleries = append(o.Galleries, galleries...)
}
@@ -312,7 +312,7 @@ func WithBackendGalleries(galls string) AppOption {
}
var galleries []Gallery
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
log.Error().Err(err).Msg("failed loading galleries")
xlog.Error("failed loading galleries", "error", err)
}
o.BackendGalleries = append(o.BackendGalleries, galleries...)
}
@@ -470,7 +470,7 @@ func WithHttpGetExemptedEndpoints(endpoints []string) AppOption {
if err == nil && r != nil {
o.HttpGetExemptedEndpoints = append(o.HttpGetExemptedEndpoints, r)
} else {
log.Warn().Err(err).Str("regex", epr).Msg("Error while compiling HTTP Get Exemption regex, skipping this entry.")
xlog.Warn("Error while compiling HTTP Get Exemption regex, skipping this entry.", "error", err, "regex", epr)
}
}
}

22
core/config/gguf.go
View File

@@ -2,7 +2,7 @@ package config
import (
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
gguf "github.com/gpustack/gguf-parser-go"
)
@@ -35,22 +35,22 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
// vram estimation
vram, err := xsysinfo.TotalAvailableVRAM()
if err != nil {
log.Error().Msgf("guessDefaultsFromFile(TotalAvailableVRAM): %s", err)
xlog.Error("guessDefaultsFromFile(TotalAvailableVRAM)", "error", err)
} else if vram > 0 {
estimate, err := xsysinfo.EstimateGGUFVRAMUsage(f, vram)
if err != nil {
log.Error().Msgf("guessDefaultsFromFile(EstimateGGUFVRAMUsage): %s", err)
xlog.Error("guessDefaultsFromFile(EstimateGGUFVRAMUsage)", "error", err)
} else {
if estimate.IsFullOffload {
log.Warn().Msgf("guessDefaultsFromFile: %s", "full offload is recommended")
xlog.Warn("guessDefaultsFromFile: full offload is recommended")
}
if estimate.EstimatedVRAM > vram {
log.Warn().Msgf("guessDefaultsFromFile: %s", "estimated VRAM usage is greater than available VRAM")
xlog.Warn("guessDefaultsFromFile: estimated VRAM usage is greater than available VRAM")
}
if cfg.NGPULayers == nil && estimate.EstimatedLayers > 0 {
log.Debug().Msgf("guessDefaultsFromFile: %d layers estimated", estimate.EstimatedLayers)
xlog.Debug("guessDefaultsFromFile: layers estimated", "layers", estimate.EstimatedLayers)
cfg.NGPULayers = &estimate.EstimatedLayers
}
}
@@ -62,20 +62,16 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
cfg.NGPULayers = &defaultHigh
}
log.Debug().Any("NGPULayers", cfg.NGPULayers).Msgf("guessDefaultsFromFile: %s", "NGPULayers set")
xlog.Debug("guessDefaultsFromFile: NGPULayers set", "NGPULayers", cfg.NGPULayers)
// template estimations
if cfg.HasTemplate() {
// nothing to guess here
log.Debug().Any("name", cfg.Name).Msgf("guessDefaultsFromFile: %s", "template already set")
xlog.Debug("guessDefaultsFromFile: template already set", "name", cfg.Name)
return
}
log.Debug().
Any("eosTokenID", f.Tokenizer().EOSTokenID).
Any("bosTokenID", f.Tokenizer().BOSTokenID).
Any("modelName", f.Metadata().Name).
Any("architecture", f.Architecture().Architecture).Msgf("Model file loaded: %s", cfg.ModelFileName())
xlog.Debug("Model file loaded", "file", cfg.ModelFileName(), "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
// guess the name
if cfg.Name == "" {

8
core/config/guesser.go
View File

@@ -5,17 +5,17 @@ import (
"path/filepath"
gguf "github.com/gpustack/gguf-parser-go"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
log.Debug().Msgf("guessDefaultsFromFile: %s", "guessing disabled with LOCALAI_DISABLE_GUESSING")
xlog.Debug("guessDefaultsFromFile: guessing disabled with LOCALAI_DISABLE_GUESSING")
return
}
if modelPath == "" {
log.Debug().Msgf("guessDefaultsFromFile: %s", "modelPath is empty")
xlog.Debug("guessDefaultsFromFile: modelPath is empty")
return
}
@@ -24,7 +24,7 @@ func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
defer func() {
if r := recover(); r != nil {
log.Error().Msgf("guessDefaultsFromFile: %s", "panic while parsing gguf file")
xlog.Error("guessDefaultsFromFile: panic while parsing gguf file")
}
}()

750
core/config/model_config_loader.go
View File

@@ -1,375 +1,375 @@
package config
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/charmbracelet/glamour"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"gopkg.in/yaml.v3"
)
type ModelConfigLoader struct {
configs map[string]ModelConfig
modelPath string
sync.Mutex
}
func NewModelConfigLoader(modelPath string) *ModelConfigLoader {
return &ModelConfigLoader{
configs: make(map[string]ModelConfig),
modelPath: modelPath,
}
}
type LoadOptions struct {
modelPath string
debug bool
threads, ctxSize int
f16 bool
}
func LoadOptionDebug(debug bool) ConfigLoaderOption {
return func(o *LoadOptions) {
o.debug = debug
}
}
func LoadOptionThreads(threads int) ConfigLoaderOption {
return func(o *LoadOptions) {
o.threads = threads
}
}
func LoadOptionContextSize(ctxSize int) ConfigLoaderOption {
return func(o *LoadOptions) {
o.ctxSize = ctxSize
}
}
func ModelPath(modelPath string) ConfigLoaderOption {
return func(o *LoadOptions) {
o.modelPath = modelPath
}
}
func LoadOptionF16(f16 bool) ConfigLoaderOption {
return func(o *LoadOptions) {
o.f16 = f16
}
}
type ConfigLoaderOption func(*LoadOptions)
func (lo *LoadOptions) Apply(options ...ConfigLoaderOption) {
for _, l := range options {
l(lo)
}
}
// TODO: either in the next PR or the next commit, I want to merge these down into a single function that looks at the first few characters of the file to determine if we need to deserialize to []BackendConfig or BackendConfig
func readMultipleModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*ModelConfig, error) {
c := &[]*ModelConfig{}
f, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot read config file %q: %w", file, err)
}
if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot unmarshal config file %q: %w", file, err)
}
for _, cc := range *c {
cc.modelConfigFile = file
cc.SetDefaults(opts...)
}
return *c, nil
}
func readModelConfigFromFile(file string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
lo := &LoadOptions{}
lo.Apply(opts...)
c := &ModelConfig{}
f, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot read config file %q: %w", file, err)
}
if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot unmarshal config file %q: %w", file, err)
}
c.SetDefaults(opts...)
c.modelConfigFile = file
return c, nil
}
// Load a config file for a model
func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
// Load a config file if present after the model name
cfg := &ModelConfig{
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: modelName,
},
},
}
cfgExisting, exists := bcl.GetModelConfig(modelName)
if exists {
cfg = &cfgExisting
} else {
// Try loading a model config file
modelConfig := filepath.Join(modelPath, modelName+".yaml")
if _, err := os.Stat(modelConfig); err == nil {
if err := bcl.ReadModelConfig(
modelConfig, opts...,
); err != nil {
return nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error())
}
cfgExisting, exists = bcl.GetModelConfig(modelName)
if exists {
cfg = &cfgExisting
}
}
}
cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
return cfg, nil
}
func (bcl *ModelConfigLoader) LoadModelConfigFileByNameDefaultOptions(modelName string, appConfig *ApplicationConfig) (*ModelConfig, error) {
return bcl.LoadModelConfigFileByName(modelName, appConfig.SystemState.Model.ModelsPath,
LoadOptionDebug(appConfig.Debug),
LoadOptionThreads(appConfig.Threads),
LoadOptionContextSize(appConfig.ContextSize),
LoadOptionF16(appConfig.F16),
ModelPath(appConfig.SystemState.Model.ModelsPath))
}
// This format is currently only used when reading a single file at startup, passed in via ApplicationConfig.ConfigFile
func (bcl *ModelConfigLoader) LoadMultipleModelConfigsSingleFile(file string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
c, err := readMultipleModelConfigsFromFile(file, opts...)
if err != nil {
return fmt.Errorf("cannot load config file: %w", err)
}
for _, cc := range c {
if valid, _ := cc.Validate(); valid {
bcl.configs[cc.Name] = *cc
}
}
return nil
}
func (bcl *ModelConfigLoader) ReadModelConfig(file string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
c, err := readModelConfigFromFile(file, opts...)
if err != nil {
return fmt.Errorf("ReadModelConfig cannot read config file %q: %w", file, err)
}
if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c
} else {
return fmt.Errorf("config is not valid")
}
return nil
}
func (bcl *ModelConfigLoader) GetModelConfig(m string) (ModelConfig, bool) {
bcl.Lock()
defer bcl.Unlock()
v, exists := bcl.configs[m]
return v, exists
}
func (bcl *ModelConfigLoader) GetAllModelsConfigs() []ModelConfig {
bcl.Lock()
defer bcl.Unlock()
var res []ModelConfig
for _, v := range bcl.configs {
res = append(res, v)
}
sort.SliceStable(res, func(i, j int) bool {
return res[i].Name < res[j].Name
})
return res
}
func (bcl *ModelConfigLoader) GetModelConfigsByFilter(filter ModelConfigFilterFn) []ModelConfig {
bcl.Lock()
defer bcl.Unlock()
var res []ModelConfig
if filter == nil {
filter = NoFilterFn
}
for n, v := range bcl.configs {
if filter(n, &v) {
res = append(res, v)
}
}
// TODO: I don't think this one needs to Sort on name... but we'll see what breaks.
return res
}
func (bcl *ModelConfigLoader) RemoveModelConfig(m string) {
bcl.Lock()
defer bcl.Unlock()
delete(bcl.configs, m)
}
// Preload prepare models if they are not local but url or huggingface repositories
func (bcl *ModelConfigLoader) Preload(modelPath string) error {
bcl.Lock()
defer bcl.Unlock()
status := func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent)
}
log.Info().Msgf("Preloading models from %s", modelPath)
renderMode := "dark"
if os.Getenv("COLOR") != "" {
renderMode = os.Getenv("COLOR")
}
glamText := func(t string) {
out, err := glamour.Render(t, renderMode)
if err == nil && os.Getenv("NO_COLOR") == "" {
fmt.Println(out)
} else {
fmt.Println(t)
}
}
for i, config := range bcl.configs {
// Download files and verify their SHA
for i, file := range config.DownloadFiles {
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename)
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
return err
}
// Create file path
filePath := filepath.Join(modelPath, file.Filename)
if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil {
return err
}
}
// If the model is an URL, expand it, and download the file
if config.IsModelURL() {
modelFileName := config.ModelFileName()
uri := downloader.URI(config.Model)
if uri.ResolveURL() != config.Model {
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.PredictionOptions.Model = modelFileName
bcl.configs[i] = *c
}
}
if config.IsMMProjURL() {
modelFileName := config.MMProjFileName()
uri := downloader.URI(config.MMProj)
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.MMProj = modelFileName
bcl.configs[i] = *c
}
if bcl.configs[i].Name != "" {
glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name))
}
if bcl.configs[i].Description != "" {
//glamText("**Description**")
glamText(bcl.configs[i].Description)
}
if bcl.configs[i].Usage != "" {
//glamText("**Usage**")
glamText(bcl.configs[i].Usage)
}
}
return nil
}
// LoadModelConfigsFromPath reads all the configurations of the models from a path
// (non-recursive)
func (bcl *ModelConfigLoader) LoadModelConfigsFromPath(path string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
entries, err := os.ReadDir(path)
if err != nil {
return fmt.Errorf("LoadModelConfigsFromPath cannot read directory '%s': %w", path, err)
}
files := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
return err
}
files = append(files, info)
}
for _, file := range files {
// Skip templates, YAML and .keep files
if !strings.Contains(file.Name(), ".yaml") && !strings.Contains(file.Name(), ".yml") ||
strings.HasPrefix(file.Name(), ".") {
continue
}
c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...)
if err != nil {
log.Error().Err(err).Str("File Name", file.Name()).Msgf("LoadModelConfigsFromPath cannot read config file")
continue
}
if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c
} else {
log.Error().Err(err).Str("Name", c.Name).Msgf("config is not valid")
}
}
return nil
}
package config
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/charmbracelet/glamour"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/xlog"
"gopkg.in/yaml.v3"
)
type ModelConfigLoader struct {
configs map[string]ModelConfig
modelPath string
sync.Mutex
}
func NewModelConfigLoader(modelPath string) *ModelConfigLoader {
return &ModelConfigLoader{
configs: make(map[string]ModelConfig),
modelPath: modelPath,
}
}
type LoadOptions struct {
modelPath string
debug bool
threads, ctxSize int
f16 bool
}
func LoadOptionDebug(debug bool) ConfigLoaderOption {
return func(o *LoadOptions) {
o.debug = debug
}
}
func LoadOptionThreads(threads int) ConfigLoaderOption {
return func(o *LoadOptions) {
o.threads = threads
}
}
func LoadOptionContextSize(ctxSize int) ConfigLoaderOption {
return func(o *LoadOptions) {
o.ctxSize = ctxSize
}
}
func ModelPath(modelPath string) ConfigLoaderOption {
return func(o *LoadOptions) {
o.modelPath = modelPath
}
}
func LoadOptionF16(f16 bool) ConfigLoaderOption {
return func(o *LoadOptions) {
o.f16 = f16
}
}
type ConfigLoaderOption func(*LoadOptions)
func (lo *LoadOptions) Apply(options ...ConfigLoaderOption) {
for _, l := range options {
l(lo)
}
}
// TODO: either in the next PR or the next commit, I want to merge these down into a single function that looks at the first few characters of the file to determine if we need to deserialize to []BackendConfig or BackendConfig
func readMultipleModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*ModelConfig, error) {
c := &[]*ModelConfig{}
f, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot read config file %q: %w", file, err)
}
if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readMultipleModelConfigsFromFile cannot unmarshal config file %q: %w", file, err)
}
for _, cc := range *c {
cc.modelConfigFile = file
cc.SetDefaults(opts...)
}
return *c, nil
}
func readModelConfigFromFile(file string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
lo := &LoadOptions{}
lo.Apply(opts...)
c := &ModelConfig{}
f, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot read config file %q: %w", file, err)
}
if err := yaml.Unmarshal(f, c); err != nil {
return nil, fmt.Errorf("readModelConfigFromFile cannot unmarshal config file %q: %w", file, err)
}
c.SetDefaults(opts...)
c.modelConfigFile = file
return c, nil
}
// Load a config file for a model
func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath string, opts ...ConfigLoaderOption) (*ModelConfig, error) {
// Load a config file if present after the model name
cfg := &ModelConfig{
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{
Model: modelName,
},
},
}
cfgExisting, exists := bcl.GetModelConfig(modelName)
if exists {
cfg = &cfgExisting
} else {
// Try loading a model config file
modelConfig := filepath.Join(modelPath, modelName+".yaml")
if _, err := os.Stat(modelConfig); err == nil {
if err := bcl.ReadModelConfig(
modelConfig, opts...,
); err != nil {
return nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error())
}
cfgExisting, exists = bcl.GetModelConfig(modelName)
if exists {
cfg = &cfgExisting
}
}
}
cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
return cfg, nil
}
func (bcl *ModelConfigLoader) LoadModelConfigFileByNameDefaultOptions(modelName string, appConfig *ApplicationConfig) (*ModelConfig, error) {
return bcl.LoadModelConfigFileByName(modelName, appConfig.SystemState.Model.ModelsPath,
LoadOptionDebug(appConfig.Debug),
LoadOptionThreads(appConfig.Threads),
LoadOptionContextSize(appConfig.ContextSize),
LoadOptionF16(appConfig.F16),
ModelPath(appConfig.SystemState.Model.ModelsPath))
}
// This format is currently only used when reading a single file at startup, passed in via ApplicationConfig.ConfigFile
func (bcl *ModelConfigLoader) LoadMultipleModelConfigsSingleFile(file string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
c, err := readMultipleModelConfigsFromFile(file, opts...)
if err != nil {
return fmt.Errorf("cannot load config file: %w", err)
}
for _, cc := range c {
if valid, _ := cc.Validate(); valid {
bcl.configs[cc.Name] = *cc
}
}
return nil
}
func (bcl *ModelConfigLoader) ReadModelConfig(file string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
c, err := readModelConfigFromFile(file, opts...)
if err != nil {
return fmt.Errorf("ReadModelConfig cannot read config file %q: %w", file, err)
}
if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c
} else {
return fmt.Errorf("config is not valid")
}
return nil
}
func (bcl *ModelConfigLoader) GetModelConfig(m string) (ModelConfig, bool) {
bcl.Lock()
defer bcl.Unlock()
v, exists := bcl.configs[m]
return v, exists
}
func (bcl *ModelConfigLoader) GetAllModelsConfigs() []ModelConfig {
bcl.Lock()
defer bcl.Unlock()
var res []ModelConfig
for _, v := range bcl.configs {
res = append(res, v)
}
sort.SliceStable(res, func(i, j int) bool {
return res[i].Name < res[j].Name
})
return res
}
func (bcl *ModelConfigLoader) GetModelConfigsByFilter(filter ModelConfigFilterFn) []ModelConfig {
bcl.Lock()
defer bcl.Unlock()
var res []ModelConfig
if filter == nil {
filter = NoFilterFn
}
for n, v := range bcl.configs {
if filter(n, &v) {
res = append(res, v)
}
}
// TODO: I don't think this one needs to Sort on name... but we'll see what breaks.
return res
}
func (bcl *ModelConfigLoader) RemoveModelConfig(m string) {
bcl.Lock()
defer bcl.Unlock()
delete(bcl.configs, m)
}
// Preload prepare models if they are not local but url or huggingface repositories
func (bcl *ModelConfigLoader) Preload(modelPath string) error {
bcl.Lock()
defer bcl.Unlock()
status := func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent)
}
xlog.Info("Preloading models", "path", modelPath)
renderMode := "dark"
if os.Getenv("COLOR") != "" {
renderMode = os.Getenv("COLOR")
}
glamText := func(t string) {
out, err := glamour.Render(t, renderMode)
if err == nil && os.Getenv("NO_COLOR") == "" {
fmt.Println(out)
} else {
fmt.Println(t)
}
}
for i, config := range bcl.configs {
// Download files and verify their SHA
for i, file := range config.DownloadFiles {
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
return err
}
// Create file path
filePath := filepath.Join(modelPath, file.Filename)
if err := file.URI.DownloadFile(filePath, file.SHA256, i, len(config.DownloadFiles), status); err != nil {
return err
}
}
// If the model is an URL, expand it, and download the file
if config.IsModelURL() {
modelFileName := config.ModelFileName()
uri := downloader.URI(config.Model)
if uri.ResolveURL() != config.Model {
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.PredictionOptions.Model = modelFileName
bcl.configs[i] = *c
}
}
if config.IsMMProjURL() {
modelFileName := config.MMProjFileName()
uri := downloader.URI(config.MMProj)
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, modelFileName)); errors.Is(err, os.ErrNotExist) {
err := uri.DownloadFile(filepath.Join(modelPath, modelFileName), "", 0, 0, status)
if err != nil {
return err
}
}
cc := bcl.configs[i]
c := &cc
c.MMProj = modelFileName
bcl.configs[i] = *c
}
if bcl.configs[i].Name != "" {
glamText(fmt.Sprintf("**Model name**: _%s_", bcl.configs[i].Name))
}
if bcl.configs[i].Description != "" {
//glamText("**Description**")
glamText(bcl.configs[i].Description)
}
if bcl.configs[i].Usage != "" {
//glamText("**Usage**")
glamText(bcl.configs[i].Usage)
}
}
return nil
}
// LoadModelConfigsFromPath reads all the configurations of the models from a path
// (non-recursive)
func (bcl *ModelConfigLoader) LoadModelConfigsFromPath(path string, opts ...ConfigLoaderOption) error {
bcl.Lock()
defer bcl.Unlock()
entries, err := os.ReadDir(path)
if err != nil {
return fmt.Errorf("LoadModelConfigsFromPath cannot read directory '%s': %w", path, err)
}
files := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
return err
}
files = append(files, info)
}
for _, file := range files {
// Skip templates, YAML and .keep files
if !strings.Contains(file.Name(), ".yaml") && !strings.Contains(file.Name(), ".yml") ||
strings.HasPrefix(file.Name(), ".") {
continue
}
c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...)
if err != nil {
xlog.Error("LoadModelConfigsFromPath cannot read config file", "error", err, "File Name", file.Name())
continue
}
if valid, _ := c.Validate(); valid {
bcl.configs[c.Name] = *c
} else {
xlog.Error("config is not valid", "error", err, "Name", c.Name)
}
}
return nil
}

14
core/explorer/discovery.go
View File

@@ -7,7 +7,7 @@ import (
"sync"
"time"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/core/schema"
@@ -57,21 +57,21 @@ func (s *DiscoveryServer) runBackground() {
// do not do in parallel
n, err := p2p.NewNode(token)
if err != nil {
log.Err(err).Msg("Failed to create node")
xlog.Error("Failed to create node", "error", err)
s.failedToken(token)
continue
}
err = n.Start(c)
if err != nil {
log.Err(err).Msg("Failed to start node")
xlog.Error("Failed to start node", "error", err)
s.failedToken(token)
continue
}
ledger, err := n.Ledger()
if err != nil {
log.Err(err).Msg("Failed to start ledger")
xlog.Error("Failed to start ledger", "error", err)
s.failedToken(token)
continue
}
@@ -92,10 +92,10 @@ func (s *DiscoveryServer) runBackground() {
}
}
log.Debug().Any("network", token).Msgf("Network has %d clusters", len(ledgerK))
xlog.Debug("Network clusters", "network", token, "count", len(ledgerK))
if len(ledgerK) != 0 {
for _, k := range ledgerK {
log.Debug().Any("network", token).Msgf("Clusterdata %+v", k)
xlog.Debug("Clusterdata", "network", token, "cluster", k)
}
}
@@ -128,7 +128,7 @@ func (s *DiscoveryServer) deleteFailedConnections() {
for _, t := range s.database.TokenList() {
data, _ := s.database.Get(t)
if data.Failures > s.errorThreshold {
log.Info().Any("token", t).Msg("Token has been removed from the database")
xlog.Info("Token has been removed from the database", "token", t)
s.database.Delete(t)
}
}

6
core/gallery/backend_types.go
View File

@@ -5,7 +5,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// BackendMetadata represents the metadata stored in a JSON file for each installed backend
@@ -37,11 +37,11 @@ func (backend *GalleryBackend) FindBestBackendFromMeta(systemState *system.Syste
realBackend := backend.CapabilitiesMap[systemState.Capability(backend.CapabilitiesMap)]
if realBackend == "" {
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("No backend found for reported capability")
xlog.Debug("No backend found for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
return nil
}
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("Found backend for reported capability")
xlog.Debug("Found backend for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
return backends.FindByName(realBackend)
}

26
core/gallery/backends.go
View File

@@ -16,8 +16,8 @@ import (
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
cp "github.com/otiai10/copy"
"github.com/rs/zerolog/log"
)
const (
@@ -86,7 +86,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
return fmt.Errorf("backend name is empty")
}
log.Debug().Interface("galleries", galleries).Str("name", name).Msg("Installing backend from gallery")
xlog.Debug("Installing backend from gallery", "galleries", galleries, "name", name)
backends, err := AvailableBackends(galleries, systemState)
if err != nil {
@@ -99,7 +99,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
}
if backend.IsMeta() {
log.Debug().Interface("systemState", systemState).Str("name", name).Msg("Backend is a meta backend")
xlog.Debug("Backend is a meta backend", "systemState", systemState, "name", name)
// Then, let's try to find the best backend based on the capabilities map
bestBackend := backend.FindBestBackendFromMeta(systemState, backends)
@@ -107,7 +107,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
return fmt.Errorf("no backend found with capabilities %q", backend.CapabilitiesMap)
}
log.Debug().Str("name", name).Str("bestBackend", bestBackend.Name).Msg("Installing backend from meta backend")
xlog.Debug("Installing backend from meta backend", "name", name, "bestBackend", bestBackend.Name)
// Then, let's install the best backend
if err := InstallBackend(ctx, systemState, modelLoader, bestBackend, downloadStatus); err != nil {
@@ -164,7 +164,7 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
return fmt.Errorf("failed copying: %w", err)
}
} else {
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloading backend")
xlog.Debug("Downloading backend", "uri", config.URI, "backendPath", backendPath)
if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil {
success := false
// Try to download from mirrors
@@ -177,24 +177,24 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
}
if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
success = true
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend")
xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
break
}
}
if !success {
log.Error().Str("uri", config.URI).Str("backendPath", backendPath).Err(err).Msg("Failed to download backend")
xlog.Error("Failed to download backend", "uri", config.URI, "backendPath", backendPath, "error", err)
return fmt.Errorf("failed to download backend %q: %v", config.URI, err)
}
} else {
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend")
xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
}
}
// sanity check - check if runfile is present
runFile := filepath.Join(backendPath, runFile)
if _, err := os.Stat(runFile); os.IsNotExist(err) {
log.Error().Str("runFile", runFile).Msg("Run file not found")
xlog.Error("Run file not found", "runFile", runFile)
return fmt.Errorf("not a valid backend: run file not found %q", runFile)
}
@@ -271,7 +271,7 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error
if metadata != nil && metadata.MetaBackendFor != "" {
metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor)
log.Debug().Str("backendDirectory", metaBackendDirectory).Msg("Deleting meta backend")
xlog.Debug("Deleting meta backend", "backendDirectory", metaBackendDirectory)
if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) {
return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor)
}
@@ -330,9 +330,9 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
}
}
} else if !errors.Is(err, os.ErrNotExist) {
log.Warn().Err(err).Msg("Failed to read system backends, proceeding with user-managed backends")
xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err)
} else if errors.Is(err, os.ErrNotExist) {
log.Debug().Msg("No system backends found")
xlog.Debug("No system backends found")
}
// User-managed backends and alias collection
@@ -442,7 +442,7 @@ func RegisterBackends(systemState *system.SystemState, modelLoader *model.ModelL
}
for _, backend := range backends {
log.Debug().Str("name", backend.Name).Str("runFile", backend.RunFile).Msg("Registering backend")
xlog.Debug("Registering backend", "name", backend.Name, "runFile", backend.RunFile)
modelLoader.SetExternalBackend(backend.Name, backend.RunFile)
}

8
core/gallery/gallery.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/xsync"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"gopkg.in/yaml.v2"
)
@@ -26,7 +26,7 @@ func GetGalleryConfigFromURL[T any](url string, basePath string) (T, error) {
return yaml.Unmarshal(d, &config)
})
if err != nil {
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
xlog.Error("failed to get gallery config for url", "error", err, "url", url)
return config, err
}
return config, nil
@@ -39,7 +39,7 @@ func GetGalleryConfigFromURLWithContext[T any](ctx context.Context, url string,
return yaml.Unmarshal(d, &config)
})
if err != nil {
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
xlog.Error("failed to get gallery config for url", "error", err, "url", url)
return config, err
}
return config, nil
@@ -310,7 +310,7 @@ func getGalleryElements[T GalleryElement](gallery config.Gallery, basePath strin
})
if err != nil {
if yamlErr, ok := err.(*yaml.TypeError); ok {
log.Debug().Msgf("YAML errors: %s\n\nwreckage of models: %+v", strings.Join(yamlErr.Errors, "\n"), models)
xlog.Debug("YAML errors", "errors", strings.Join(yamlErr.Errors, "\n"), "models", models)
}
return models, fmt.Errorf("failed to read gallery elements: %w", err)
}

12
core/gallery/importers/importers.go
View File

@@ -6,7 +6,7 @@ import (
"os"
"strings"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/config"
@@ -52,10 +52,10 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
if err != nil {
// maybe not a HF repository
// TODO: maybe we can check if the URI is a valid HF repository
log.Debug().Str("uri", uri).Str("hfrepoID", hfrepoID).Msg("Failed to get model details, maybe not a HF repository")
xlog.Debug("Failed to get model details, maybe not a HF repository", "uri", uri, "hfrepoID", hfrepoID)
} else {
log.Debug().Str("uri", uri).Msg("Got model details")
log.Debug().Any("details", hfDetails).Msg("Model details")
xlog.Debug("Got model details", "uri", uri)
xlog.Debug("Model details", "details", hfDetails)
}
// handle local config files ("/my-model.yaml" or "file://my-model.yaml")
@@ -73,13 +73,13 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
return nil
})
if err != nil {
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition")
xlog.Error("error reading model definition", "error", err, "filepath", localURI)
return gallery.ModelConfig{}, err
}
} else {
modelYAML, err = os.ReadFile(localURI)
if err != nil {
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition")
xlog.Error("error reading model definition", "error", err, "filepath", localURI)
return gallery.ModelConfig{}, err
}
}

8
core/gallery/importers/llama-cpp.go
View File

@@ -11,7 +11,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/functions"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"go.yaml.in/yaml/v2"
)
@@ -22,7 +22,7 @@ type LlamaCPPImporter struct{}
func (i *LlamaCPPImporter) Match(details Details) bool {
preferences, err := details.Preferences.MarshalJSON()
if err != nil {
log.Error().Err(err).Msg("failed to marshal preferences")
xlog.Error("failed to marshal preferences", "error", err)
return false
}
@@ -31,7 +31,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
if len(preferences) > 0 {
err = json.Unmarshal(preferences, &preferencesMap)
if err != nil {
log.Error().Err(err).Msg("failed to unmarshal preferences")
xlog.Error("failed to unmarshal preferences", "error", err)
return false
}
}
@@ -63,7 +63,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) {
log.Debug().Str("uri", details.URI).Msg("llama.cpp importer matched")
xlog.Debug("llama.cpp importer matched", "uri", details.URI)
preferences, err := details.Preferences.MarshalJSON()
if err != nil {

34
core/gallery/models.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"gopkg.in/yaml.v3"
)
@@ -131,9 +131,9 @@ func InstallModelFromGallery(
if err != nil {
return err
}
log.Debug().Msgf("Installed model %q", installedModel.Name)
xlog.Debug("Installed model", "model", installedModel.Name)
if automaticallyInstallBackend && installedModel.Backend != "" {
log.Debug().Msgf("Installing backend %q", installedModel.Backend)
xlog.Debug("Installing backend", "backend", installedModel.Backend)
if err := InstallBackendFromGallery(ctx, backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil {
return err
@@ -165,7 +165,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
}
if len(configOverrides) > 0 {
log.Debug().Msgf("Config overrides %+v", configOverrides)
xlog.Debug("Config overrides", "overrides", configOverrides)
}
// Download files and verify their SHA
@@ -177,7 +177,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
default:
}
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename)
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
return nil, err
@@ -189,7 +189,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
if enforceScan {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", config.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return nil, err
}
}
@@ -218,7 +218,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
}
log.Debug().Msgf("Prompt template %q written", template.Name)
xlog.Debug("Prompt template written", "template", template.Name)
}
name := config.Name
@@ -269,7 +269,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, fmt.Errorf("failed to write updated config file: %v", err)
}
log.Debug().Msgf("Written config file %s", configFilePath)
xlog.Debug("Written config file", "file", configFilePath)
}
// Save the model gallery file for further reference
@@ -279,7 +279,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
return nil, err
}
log.Debug().Msgf("Written gallery file %s", modelFile)
xlog.Debug("Written gallery file", "file", modelFile)
return &modelConfig, os.WriteFile(modelFile, data, 0600)
}
@@ -341,7 +341,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
allFiles = append(allFiles, fullPath)
}
} else {
log.Error().Err(err).Msgf("failed to read gallery file %s", configFile)
xlog.Error("failed to read gallery file", "error", err, "file", configFile)
}
for _, f := range additionalFiles {
@@ -391,26 +391,26 @@ func DeleteModelFromSystem(systemState *system.SystemState, name string) error {
name := strings.TrimSuffix(f.Name(), ".yaml")
name = strings.TrimSuffix(name, ".yml")
log.Debug().Msgf("Checking file %s", f.Name())
xlog.Debug("Checking file", "file", f.Name())
files, err := listModelFiles(systemState, name)
if err != nil {
log.Debug().Err(err).Msgf("failed to list files for model %s", f.Name())
xlog.Debug("failed to list files for model", "error", err, "model", f.Name())
continue
}
allOtherFiles = append(allOtherFiles, files...)
}
log.Debug().Msgf("Files to remove: %+v", filesToRemove)
log.Debug().Msgf("All other files: %+v", allOtherFiles)
xlog.Debug("Files to remove", "files", filesToRemove)
xlog.Debug("All other files", "files", allOtherFiles)
// Removing files
for _, f := range filesToRemove {
if slices.Contains(allOtherFiles, f) {
log.Debug().Msgf("Skipping file %s because it is part of another model", f)
xlog.Debug("Skipping file because it is part of another model", "file", f)
continue
}
if e := os.Remove(f); e != nil {
log.Error().Err(e).Msgf("failed to remove file %s", f)
xlog.Error("failed to remove file", "error", e, "file", f)
}
}
@@ -436,7 +436,7 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
for _, file := range galleryModel.AdditionalFiles {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", galleryModel.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
xlog.Error("Contains unsafe file(s)!", "model", galleryModel.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
return err
}
}

15
core/http/app.go
View File

@@ -21,7 +21,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// Embed a directory
@@ -101,18 +101,13 @@ func API(application *application.Application) (*echo.Echo, error) {
})
}
// Custom logger middleware using zerolog
// Custom logger middleware using xlog
e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
req := c.Request()
res := c.Response()
start := log.Logger.Info()
err := next(c)
start.
Str("method", req.Method).
Str("path", req.URL.Path).
Int("status", res.Status).
Msg("HTTP request")
xlog.Info("HTTP request", "method", req.Method, "path", req.URL.Path, "status", res.Status)
return err
}
})
@@ -193,7 +188,7 @@ func API(application *application.Application) (*echo.Echo, error) {
// CSRF middleware
if application.ApplicationConfig().CSRF {
log.Debug().Msg("Enabling CSRF middleware. Tokens are now required for state-modifying requests")
xlog.Debug("Enabling CSRF middleware. Tokens are now required for state-modifying requests")
e.Use(middleware.CSRF())
}
@@ -219,7 +214,7 @@ func API(application *application.Application) (*echo.Echo, error) {
// Log startup message
e.Server.RegisterOnShutdown(func() {
log.Info().Msg("LocalAI API server shutting down")
xlog.Info("LocalAI API server shutting down")
})
return e, nil

10
core/http/app_test.go
View File

@@ -25,8 +25,8 @@ import (
. "github.com/onsi/gomega"
"gopkg.in/yaml.v3"
"github.com/mudler/xlog"
openaigo "github.com/otiai10/openaigo"
"github.com/rs/zerolog/log"
"github.com/sashabaranov/go-openai"
"github.com/sashabaranov/go-openai/jsonschema"
)
@@ -378,7 +378,7 @@ var _ = Describe("API test", func() {
go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error")
xlog.Error("server error", "error", err)
}
}()
@@ -710,7 +710,7 @@ parameters:
go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error")
xlog.Error("server error", "error", err)
}
}()
@@ -924,7 +924,7 @@ parameters:
Expect(err).ToNot(HaveOccurred())
go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error")
xlog.Error("server error", "error", err)
}
}()
@@ -1405,7 +1405,7 @@ parameters:
go func() {
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
log.Error().Err(err).Msg("server error")
xlog.Error("server error", "error", err)
}
}()

4
core/http/endpoints/elevenlabs/soundgeneration.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// SoundGenerationEndpoint is the ElevenLabs SoundGeneration endpoint https://elevenlabs.io/docs/api-reference/sound-generation
@@ -30,7 +30,7 @@ func SoundGenerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader
return echo.ErrBadRequest
}
log.Debug().Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Sound Generation Request about to be sent to backend")
xlog.Debug("Sound Generation Request about to be sent to backend", "modelFile", "modelFile", "backend", cfg.Backend)
// TODO: Support uploading files?
filePath, _, err := backend.SoundGeneration(input.Text, input.Duration, input.Temperature, input.DoSample, nil, nil, ml, appConfig, *cfg)

4
core/http/endpoints/elevenlabs/tts.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
@@ -33,7 +33,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest
}
log.Debug().Str("modelName", input.ModelID).Msg("elevenlabs TTS request received")
xlog.Debug("elevenlabs TTS request received", "modelName", input.ModelID)
filePath, _, err := backend.ModelTTS(input.Text, voiceID, input.LanguageCode, ml, appConfig, *cfg)
if err != nil {

4
core/http/endpoints/jina/rerank.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// JINARerankEndpoint acts like the Jina reranker endpoint (https://jina.ai/reranker/)
@@ -31,7 +31,7 @@ func JINARerankEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return echo.ErrBadRequest
}
log.Debug().Str("model", input.Model).Msg("JINA Rerank Request received")
xlog.Debug("JINA Rerank Request received", "model", input.Model)
var requestTopN int32
docs := int32(len(input.Documents))
if input.TopN == nil { // omit top_n to get all

6
core/http/endpoints/localai/backend.go
View File

@@ -4,15 +4,15 @@ import (
"encoding/json"
"fmt"
"github.com/labstack/echo/v4"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type BackendEndpointService struct {
@@ -131,7 +131,7 @@ func (mgs *BackendEndpointService) ListBackendsEndpoint(systemState *system.Syst
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc {
return func(c echo.Context) error {
log.Debug().Msgf("Listing backend galleries %+v", mgs.galleries)
xlog.Debug("Listing backend galleries", "galleries", mgs.galleries)
dat, err := json.Marshal(mgs.galleries)
if err != nil {
return err

4
core/http/endpoints/localai/detection.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// DetectionEndpoint is the LocalAI Detection endpoint https://localai.io/docs/api-reference/detection
@@ -29,7 +29,7 @@ func DetectionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appC
return echo.ErrBadRequest
}
log.Debug().Str("image", input.Image).Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Detection")
xlog.Debug("Detection", "image", input.Image, "modelFile", "modelFile", "backend", cfg.Backend)
image, err := utils.GetContentURIAsBase64(input.Image)
if err != nil {

12
core/http/endpoints/localai/gallery.go
View File

@@ -4,15 +4,15 @@ import (
"encoding/json"
"fmt"
"github.com/labstack/echo/v4"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type ModelGalleryEndpointService struct {
@@ -121,11 +121,11 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
models, err := gallery.AvailableGalleryModels(mgs.galleries, systemState)
if err != nil {
log.Error().Err(err).Msg("could not list models from galleries")
xlog.Error("could not list models from galleries", "error", err)
return err
}
log.Debug().Msgf("Available %d models from %d galleries\n", len(models), len(mgs.galleries))
xlog.Debug("Available models from galleries", "modelCount", len(models), "galleryCount", len(mgs.galleries))
m := []gallery.Metadata{}
@@ -133,7 +133,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
m = append(m, mm.Metadata)
}
log.Debug().Msgf("Models %#v", m)
xlog.Debug("Models", "models", m)
dat, err := json.Marshal(m)
if err != nil {
@@ -150,7 +150,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
func (mgs *ModelGalleryEndpointService) ListModelGalleriesEndpoint() echo.HandlerFunc {
return func(c echo.Context) error {
log.Debug().Msgf("Listing model galleries %+v", mgs.galleries)
xlog.Debug("Listing model galleries", "galleries", mgs.galleries)
dat, err := json.Marshal(mgs.galleries)
if err != nil {
return err

10
core/http/endpoints/localai/get_token_metrics.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/model"
)
@@ -34,19 +34,19 @@ func TokenMetricsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a
modelFile, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if !ok || modelFile != "" {
modelFile = input.Model
log.Warn().Msgf("Model not found in context: %s", input.Model)
xlog.Warn("Model not found in context", "model", input.Model)
}
cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(modelFile, appConfig)
if err != nil {
log.Err(err)
xlog.Error("Error loading model config", "error", err)
modelFile = input.Model
log.Warn().Msgf("Model not found in context: %s", input.Model)
xlog.Warn("Model not found in context", "model", input.Model)
} else {
modelFile = cfg.Model
}
log.Debug().Msgf("Token Metrics for model: %s", modelFile)
xlog.Debug("Token Metrics for model", "model", modelFile)
response, err := backend.TokenMetrics(modelFile, ml, appConfig, *cfg)
if err != nil {

24
core/http/endpoints/localai/mcp.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/cogito"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// MCP SSE Event Types
@@ -138,19 +138,19 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
cogitoOpts = append(
cogitoOpts,
cogito.WithStatusCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s)
xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
}),
cogito.WithReasoningCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s)
xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
}),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).Msg("[model agent] Tool call")
xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
return cogito.ToolCallDecision{
Approved: true,
}
}),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).Msg("[model agent] Tool call result")
xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
}),
)
@@ -176,7 +176,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)
@@ -279,7 +279,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
select {
case <-ctx.Done():
// Context was cancelled (client disconnected or request cancelled)
log.Debug().Msgf("Request context cancelled, stopping stream")
xlog.Debug("Request context cancelled, stopping stream")
cancel()
break LOOP
case event := <-events:
@@ -289,13 +289,13 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
}
eventData, err := json.Marshal(event)
if err != nil {
log.Debug().Msgf("Failed to marshal event: %v", err)
xlog.Debug("Failed to marshal event", "error", err)
continue
}
log.Debug().Msgf("Sending event: %s", string(eventData))
xlog.Debug("Sending event", "event", string(eventData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(eventData))
if err != nil {
log.Debug().Msgf("Sending event failed: %v", err)
xlog.Debug("Sending event failed", "error", err)
cancel()
return err
}
@@ -307,7 +307,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
c.Response().Flush()
break LOOP
}
log.Error().Msgf("Stream ended with error: %v", err)
xlog.Error("Stream ended with error", "error", err)
errorEvent := MCPErrorEvent{
Type: "error",
Message: err.Error(),
@@ -324,7 +324,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
}
}
log.Debug().Msgf("Stream ended")
xlog.Debug("Stream ended")
return nil
}
}

12
core/http/endpoints/localai/settings.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// GetSettingsEndpoint returns current settings with precedence (env > file > defaults)
@@ -118,7 +118,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
if watchdogChanged {
if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled {
if err := app.StopWatchdog(); err != nil {
log.Error().Err(err).Msg("Failed to stop watchdog")
xlog.Error("Failed to stop watchdog", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false,
Error: "Settings saved but failed to stop watchdog: " + err.Error(),
@@ -126,7 +126,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
}
} else {
if err := app.RestartWatchdog(); err != nil {
log.Error().Err(err).Msg("Failed to restart watchdog")
xlog.Error("Failed to restart watchdog", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false,
Error: "Settings saved but failed to restart watchdog: " + err.Error(),
@@ -138,7 +138,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
// Restart agent job service if retention days changed
if agentJobChanged {
if err := app.RestartAgentJobService(); err != nil {
log.Error().Err(err).Msg("Failed to restart agent job service")
xlog.Error("Failed to restart agent job service", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false,
Error: "Settings saved but failed to restart agent job service: " + err.Error(),
@@ -151,7 +151,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
if p2pChanged {
if settings.P2PToken != nil && *settings.P2PToken == "" {
if err := app.StopP2P(); err != nil {
log.Error().Err(err).Msg("Failed to stop P2P")
xlog.Error("Failed to stop P2P", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false,
Error: "Settings saved but failed to stop P2P: " + err.Error(),
@@ -164,7 +164,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
appConfig.P2PToken = token
}
if err := app.RestartP2P(); err != nil {
log.Error().Err(err).Msg("Failed to restart P2P")
xlog.Error("Failed to restart P2P", "error", err)
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
Success: false,
Error: "Settings saved but failed to restart P2P: " + err.Error(),

4
core/http/endpoints/localai/tts.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/utils"
)
@@ -36,7 +36,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest
}
log.Debug().Str("model", input.Model).Msg("LocalAI TTS Request received")
xlog.Debug("LocalAI TTS Request received", "model", input.Model)
if cfg.Backend == "" && input.Backend != "" {
cfg.Backend = input.Backend

4
core/http/endpoints/localai/vad.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// VADEndpoint is Voice-Activation-Detection endpoint
@@ -28,7 +28,7 @@ func VADEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
return echo.ErrBadRequest
}
log.Debug().Str("model", input.Model).Msg("LocalAI VAD Request received")
xlog.Debug("LocalAI VAD Request received", "model", input.Model)
resp, err := backend.VAD(input, c.Request().Context(), ml, appConfig, *cfg)

10
core/http/endpoints/localai/video.go
View File

@@ -22,7 +22,7 @@ import (
"github.com/mudler/LocalAI/core/backend"
model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func downloadFile(url string) (string, error) {
@@ -69,13 +69,13 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
return func(c echo.Context) error {
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.VideoRequest)
if !ok || input.Model == "" {
log.Error().Msg("Video Endpoint - Invalid Input")
xlog.Error("Video Endpoint - Invalid Input")
return echo.ErrBadRequest
}
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || config == nil {
log.Error().Msg("Video Endpoint - Invalid Config")
xlog.Error("Video Endpoint - Invalid Config")
return echo.ErrBadRequest
}
@@ -124,7 +124,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
defer os.RemoveAll(src)
}
log.Debug().Msgf("Parameter Config: %+v", config)
xlog.Debug("Parameter Config", "config", config)
switch config.Backend {
case "stablediffusion":
@@ -217,7 +217,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)

14
core/http/endpoints/mcp/tools.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/signals"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type sessionCache struct {
@@ -47,7 +47,7 @@ func SessionsFromMCPConfig(
// Get the list of all the tools that the Agent will be esposed to
for _, server := range remote.Servers {
log.Debug().Msgf("[MCP remote server] Configuration : %+v", server)
xlog.Debug("[MCP remote server] Configuration", "server", server)
// Create HTTP client with custom roundtripper for bearer token injection
httpClient := &http.Client{
Timeout: 360 * time.Second,
@@ -57,16 +57,16 @@ func SessionsFromMCPConfig(
transport := &mcp.StreamableClientTransport{Endpoint: server.URL, HTTPClient: httpClient}
mcpSession, err := client.Connect(ctx, transport, nil)
if err != nil {
log.Error().Err(err).Msgf("Failed to connect to MCP server %s", server.URL)
xlog.Error("Failed to connect to MCP server", "error", err, "url", server.URL)
continue
}
log.Debug().Msgf("[MCP remote server] Connected to MCP server %s", server.URL)
xlog.Debug("[MCP remote server] Connected to MCP server", "url", server.URL)
cache.cache[name] = append(cache.cache[name], mcpSession)
allSessions = append(allSessions, mcpSession)
}
for _, server := range stdio.Servers {
log.Debug().Msgf("[MCP stdio server] Configuration : %+v", server)
xlog.Debug("[MCP stdio server] Configuration", "server", server)
command := exec.Command(server.Command, server.Args...)
command.Env = os.Environ()
for key, value := range server.Env {
@@ -75,10 +75,10 @@ func SessionsFromMCPConfig(
transport := &mcp.CommandTransport{Command: command}
mcpSession, err := client.Connect(ctx, transport, nil)
if err != nil {
log.Error().Err(err).Msgf("Failed to start MCP server %s", command)
xlog.Error("Failed to start MCP server", "error", err, "command", command)
continue
}
log.Debug().Msgf("[MCP stdio server] Connected to MCP server %s", command)
xlog.Debug("[MCP stdio server] Connected to MCP server", "command", command)
cache.cache[name] = append(cache.cache[name], mcpSession)
allSessions = append(allSessions, mcpSession)
}

62
core/http/endpoints/openai/chat.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create
@@ -78,7 +78,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
textContentToReturn = functions.ParseTextContent(result, config.FunctionsConfig)
result = functions.CleanupLLMResult(result, config.FunctionsConfig)
functionResults := functions.ParseFunctionCall(result, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
xlog.Debug("Text content to return", "text", textContentToReturn)
noActionToRun := len(functionResults) > 0 && functionResults[0].Name == noAction || len(functionResults) == 0
switch {
@@ -94,7 +94,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
result, err := handleQuestion(config, cl, req, ml, startupOptions, functionResults, result, prompt)
if err != nil {
log.Error().Err(err).Msg("error handling question")
xlog.Error("error handling question", "error", err)
return err
}
usage := schema.OpenAIUsage{
@@ -195,7 +195,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
return echo.ErrBadRequest
}
log.Debug().Msgf("Chat endpoint configuration read: %+v", config)
xlog.Debug("Chat endpoint configuration read", "config", config)
funcs := input.Functions
shouldUseFn := len(input.Functions) > 0 && config.ShouldUseFunctions()
@@ -252,7 +252,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil {
input.Grammar = g
} else {
log.Error().Err(err).Msg("Failed generating grammar")
xlog.Error("Failed generating grammar", "error", err)
}
}
}
@@ -260,7 +260,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
config.Grammar = input.Grammar
if shouldUseFn {
log.Debug().Msgf("Response needs to process functions")
xlog.Debug("Response needs to process functions")
}
switch {
@@ -294,14 +294,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil {
config.Grammar = g
} else {
log.Error().Err(err).Msg("Failed generating grammar")
xlog.Error("Failed generating grammar", "error", err)
}
case input.JSONFunctionGrammarObject != nil:
g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarOptions()...)
if err == nil {
config.Grammar = g
} else {
log.Error().Err(err).Msg("Failed generating grammar")
xlog.Error("Failed generating grammar", "error", err)
}
default:
@@ -316,7 +316,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
// functions are not supported in stream mode (yet?)
toStream := input.Stream
log.Debug().Msgf("Parameters: %+v", config)
xlog.Debug("Parameters", "config", config)
var predInput string
@@ -325,16 +325,16 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if !config.TemplateConfig.UseTokenizerTemplate {
predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
log.Debug().Msgf("Prompt (after templating): %s", predInput)
xlog.Debug("Prompt (after templating)", "prompt", predInput)
if config.Grammar != "" {
log.Debug().Msgf("Grammar: %+v", config.Grammar)
xlog.Debug("Grammar", "grammar", config.Grammar)
}
}
switch {
case toStream:
log.Debug().Msgf("Stream request received")
xlog.Debug("Stream request received")
c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().Header().Set("Cache-Control", "no-cache")
c.Response().Header().Set("Connection", "keep-alive")
@@ -359,12 +359,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
select {
case <-input.Context.Done():
// Context was cancelled (client disconnected or request cancelled)
log.Debug().Msgf("Request context cancelled, stopping stream")
xlog.Debug("Request context cancelled, stopping stream")
input.Cancel()
break LOOP
case ev := <-responses:
if len(ev.Choices) == 0 {
log.Debug().Msgf("No choices in the response, skipping")
xlog.Debug("No choices in the response, skipping")
continue
}
usage = &ev.Usage // Copy a pointer to the latest usage chunk so that the stop message can reference it
@@ -373,14 +373,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
}
respData, err := json.Marshal(ev)
if err != nil {
log.Debug().Msgf("Failed to marshal response: %v", err)
xlog.Debug("Failed to marshal response", "error", err)
input.Cancel()
continue
}
log.Debug().Msgf("Sending chunk: %s", string(respData))
xlog.Debug("Sending chunk", "chunk", string(respData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
if err != nil {
log.Debug().Msgf("Sending chunk failed: %v", err)
xlog.Debug("Sending chunk failed", "error", err)
input.Cancel()
return err
}
@@ -389,7 +389,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
if err == nil {
break LOOP
}
log.Error().Msgf("Stream ended with error: %v", err)
xlog.Error("Stream ended with error", "error", err)
stopReason := FinishReasonStop
resp := &schema.OpenAIResponse{
@@ -407,7 +407,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
}
respData, marshalErr := json.Marshal(resp)
if marshalErr != nil {
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
xlog.Error("Failed to marshal error response", "error", marshalErr)
// Send a simple error message as fallback
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
} else {
@@ -445,7 +445,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData)
fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n")
c.Response().Flush()
log.Debug().Msgf("Stream ended")
xlog.Debug("Stream ended")
return nil
// no streaming mode
@@ -462,14 +462,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
xlog.Debug("Text content to return", "text", textContentToReturn)
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
switch {
case noActionsToRun:
result, err := handleQuestion(config, cl, input, ml, startupOptions, results, s, predInput)
if err != nil {
log.Error().Err(err).Msg("error handling question")
xlog.Error("error handling question", "error", err)
return
}
@@ -562,7 +562,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
Usage: usage,
}
respData, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", respData)
xlog.Debug("Response", "response", string(respData))
// Return the prediction in the response body
return c.JSON(200, resp)
@@ -573,12 +573,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, input *schema.OpenAIRequest, ml *model.ModelLoader, o *config.ApplicationConfig, funcResults []functions.FuncCallResults, result, prompt string) (string, error) {
if len(funcResults) == 0 && result != "" {
log.Debug().Msgf("nothing function results but we had a message from the LLM")
xlog.Debug("nothing function results but we had a message from the LLM")
return result, nil
}
log.Debug().Msgf("nothing to do, computing a reply")
xlog.Debug("nothing to do, computing a reply")
arg := ""
if len(funcResults) > 0 {
arg = funcResults[0].Arguments
@@ -586,23 +586,23 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
// If there is a message that the LLM already sends as part of the JSON reply, use it
arguments := map[string]interface{}{}
if err := json.Unmarshal([]byte(arg), &arguments); err != nil {
log.Debug().Msg("handleQuestion: function result did not contain a valid JSON object")
xlog.Debug("handleQuestion: function result did not contain a valid JSON object")
}
m, exists := arguments["message"]
if exists {
switch message := m.(type) {
case string:
if message != "" {
log.Debug().Msgf("Reply received from LLM: %s", message)
xlog.Debug("Reply received from LLM", "message", message)
message = backend.Finetune(*config, prompt, message)
log.Debug().Msgf("Reply received from LLM(finetuned): %s", message)
xlog.Debug("Reply received from LLM(finetuned)", "message", message)
return message, nil
}
}
}
log.Debug().Msgf("No action received from LLM, without a message, computing a reply")
xlog.Debug("No action received from LLM, without a message, computing a reply")
// Otherwise ask the LLM to understand the JSON output and the context, and return a message
// Note: This costs (in term of CPU/GPU) another computation
config.Grammar = ""
@@ -662,13 +662,13 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
predFunc, err := backend.ModelInference(input.Context, prompt, input.Messages, images, videos, audios, ml, config, cl, o, nil, toolsJSON, toolChoiceJSON, logprobs, topLogprobs, logitBias)
if err != nil {
log.Error().Err(err).Msg("model inference failed")
xlog.Error("model inference failed", "error", err)
return "", err
}
prediction, err := predFunc()
if err != nil {
log.Error().Err(err).Msg("prediction failed")
xlog.Error("prediction failed", "error", err)
return "", err
}
return backend.Finetune(*config, prompt, prediction.Response), nil

24
core/http/endpoints/openai/completion.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
@@ -52,7 +52,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
Object: "text_completion",
Usage: usage,
}
log.Debug().Msgf("Sending goroutine: %s", s)
xlog.Debug("Sending goroutine", "text", s)
responses <- resp
return true
@@ -94,10 +94,10 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
config.Grammar = input.Grammar
log.Debug().Msgf("Parameter Config: %+v", config)
xlog.Debug("Parameter Config", "config", config)
if input.Stream {
log.Debug().Msgf("Stream request received")
xlog.Debug("Stream request received")
c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().Header().Set("Cache-Control", "no-cache")
c.Response().Header().Set("Connection", "keep-alive")
@@ -116,7 +116,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
})
if err == nil {
predInput = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", predInput)
xlog.Debug("Template found, input modified", "input", predInput)
}
responses := make(chan schema.OpenAIResponse)
@@ -131,16 +131,16 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
select {
case ev := <-responses:
if len(ev.Choices) == 0 {
log.Debug().Msgf("No choices in the response, skipping")
xlog.Debug("No choices in the response, skipping")
continue
}
respData, err := json.Marshal(ev)
if err != nil {
log.Debug().Msgf("Failed to marshal response: %v", err)
xlog.Debug("Failed to marshal response", "error", err)
continue
}
log.Debug().Msgf("Sending chunk: %s", string(respData))
xlog.Debug("Sending chunk", "chunk", string(respData))
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
if err != nil {
return err
@@ -150,7 +150,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
if err == nil {
break LOOP
}
log.Error().Msgf("Stream ended with error: %v", err)
xlog.Error("Stream ended with error", "error", err)
stopReason := FinishReasonStop
errorResp := schema.OpenAIResponse{
@@ -168,7 +168,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
}
errorData, marshalErr := json.Marshal(errorResp)
if marshalErr != nil {
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
xlog.Error("Failed to marshal error response", "error", marshalErr)
// Send a simple error message as fallback
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
} else {
@@ -213,7 +213,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
})
if err == nil {
i = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", i)
xlog.Debug("Template found, input modified", "input", i)
}
r, tokenUsage, err := ComputeChoices(
@@ -250,7 +250,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)

10
core/http/endpoints/openai/edit.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// EditEndpoint is the OpenAI edit API endpoint
@@ -39,8 +39,8 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
return echo.ErrBadRequest
}
log.Debug().Msgf("Edit Endpoint Input : %+v", input)
log.Debug().Msgf("Edit Endpoint Config: %+v", *config)
xlog.Debug("Edit Endpoint Input", "input", input)
xlog.Debug("Edit Endpoint Config", "config", *config)
var result []schema.Choice
totalTokenUsage := backend.TokenUsage{}
@@ -55,7 +55,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
})
if err == nil {
i = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", i)
xlog.Debug("Template found, input modified", "input", i)
}
r, tokenUsage, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
@@ -95,7 +95,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)

6
core/http/endpoints/openai/embeddings.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/google/uuid"
"github.com/mudler/LocalAI/core/schema"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings
@@ -33,7 +33,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return echo.ErrBadRequest
}
log.Debug().Msgf("Parameter Config: %+v", config)
xlog.Debug("Parameter Config", "config", config)
items := []schema.Item{}
for i, s := range config.InputToken {
@@ -75,7 +75,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)

22
core/http/endpoints/openai/image.go
View File

@@ -23,7 +23,7 @@ import (
"github.com/mudler/LocalAI/core/backend"
model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func downloadFile(url string) (string, error) {
@@ -70,13 +70,13 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
return func(c echo.Context) error {
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
if !ok || input.Model == "" {
log.Error().Msg("Image Endpoint - Invalid Input")
xlog.Error("Image Endpoint - Invalid Input")
return echo.ErrBadRequest
}
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || config == nil {
log.Error().Msg("Image Endpoint - Invalid Config")
xlog.Error("Image Endpoint - Invalid Config")
return echo.ErrBadRequest
}
@@ -113,7 +113,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
}
}
log.Debug().Msgf("Parameter Config: %+v", config)
xlog.Debug("Parameter Config", "config", config)
switch config.Backend {
case "stablediffusion":
@@ -124,7 +124,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
if !strings.Contains(input.Size, "x") {
input.Size = "512x512"
log.Warn().Msgf("Invalid size, using default 512x512")
xlog.Warn("Invalid size, using default 512x512")
}
sizeParts := strings.Split(input.Size, "x")
@@ -235,7 +235,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)
@@ -251,21 +251,21 @@ func processImageFile(file string, generatedContentDir string) string {
if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") {
out, err := downloadFile(file)
if err != nil {
log.Error().Err(err).Msgf("Failed downloading file: %s", file)
xlog.Error("Failed downloading file", "error", err, "file", file)
return ""
}
defer os.RemoveAll(out)
fileData, err = os.ReadFile(out)
if err != nil {
log.Error().Err(err).Msgf("Failed reading downloaded file: %s", out)
xlog.Error("Failed reading downloaded file", "error", err, "file", out)
return ""
}
} else {
// base 64 decode the file and write it somewhere that we will cleanup
fileData, err = base64.StdEncoding.DecodeString(file)
if err != nil {
log.Error().Err(err).Msgf("Failed decoding base64 file")
xlog.Error("Failed decoding base64 file", "error", err)
return ""
}
}
@@ -273,7 +273,7 @@ func processImageFile(file string, generatedContentDir string) string {
// Create a temporary file
outputFile, err := os.CreateTemp(generatedContentDir, "b64")
if err != nil {
log.Error().Err(err).Msg("Failed creating temporary file")
xlog.Error("Failed creating temporary file", "error", err)
return ""
}
@@ -282,7 +282,7 @@ func processImageFile(file string, generatedContentDir string) string {
_, err = writer.Write(fileData)
if err != nil {
outputFile.Close()
log.Error().Err(err).Msg("Failed writing to temporary file")
xlog.Error("Failed writing to temporary file", "error", err)
return ""
}
outputFile.Close()

34
core/http/endpoints/openai/inpainting.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/backend"
"github.com/mudler/LocalAI/core/config"
@@ -48,7 +48,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
stepsStr := c.FormValue("steps")
if modelName == "" || prompt == "" {
log.Error().Msg("Inpainting Endpoint - missing model or prompt")
xlog.Error("Inpainting Endpoint - missing model or prompt")
return echo.ErrBadRequest
}
@@ -63,12 +63,12 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// Get uploaded files
imageFile, err := c.FormFile("image")
if err != nil {
log.Error().Err(err).Msg("Inpainting Endpoint - missing image file")
xlog.Error("Inpainting Endpoint - missing image file", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "missing image file")
}
maskFile, err := c.FormFile("mask")
if err != nil {
log.Error().Err(err).Msg("Inpainting Endpoint - missing mask file")
xlog.Error("Inpainting Endpoint - missing mask file", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "missing mask file")
}
@@ -100,7 +100,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// get model config from context (middleware set it)
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
if !ok || cfg == nil {
log.Error().Msg("Inpainting Endpoint - model config not found in context")
xlog.Error("Inpainting Endpoint - model config not found in context")
return echo.ErrBadRequest
}
@@ -109,7 +109,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
tmpDir := appConfig.GeneratedContentDir
// Ensure the directory exists
if err := os.MkdirAll(tmpDir, 0750); err != nil {
log.Error().Err(err).Msgf("Inpainting Endpoint - failed to create generated content dir: %s", tmpDir)
xlog.Error("Inpainting Endpoint - failed to create generated content dir", "error", err, "dir", tmpDir)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to prepare storage")
}
id := uuid.New().String()
@@ -132,32 +132,32 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
// Best-effort cleanup; log any failures
if jf != nil {
if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file in cleanup")
xlog.Warn("Inpainting Endpoint - failed to close temp json file in cleanup", "error", cerr)
}
if name := jf.Name(); name != "" {
if rerr := os.Remove(name); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove temp json file %s in cleanup", name)
xlog.Warn("Inpainting Endpoint - failed to remove temp json file in cleanup", "error", rerr, "file", name)
}
}
}
if jsonPath != "" {
if rerr := os.Remove(jsonPath); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove json file %s in cleanup", jsonPath)
xlog.Warn("Inpainting Endpoint - failed to remove json file in cleanup", "error", rerr, "file", jsonPath)
}
}
if dst != "" {
if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove dst file %s in cleanup", dst)
xlog.Warn("Inpainting Endpoint - failed to remove dst file in cleanup", "error", rerr, "file", dst)
}
}
if origRef != "" {
if rerr := os.Remove(origRef); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove orig ref file %s in cleanup", origRef)
xlog.Warn("Inpainting Endpoint - failed to remove orig ref file in cleanup", "error", rerr, "file", origRef)
}
}
if maskRef != "" {
if rerr := os.Remove(maskRef); rerr != nil && !os.IsNotExist(rerr) {
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove mask ref file %s in cleanup", maskRef)
xlog.Warn("Inpainting Endpoint - failed to remove mask ref file in cleanup", "error", rerr, "file", maskRef)
}
}
}
@@ -175,7 +175,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err
}
if cerr := origTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close orig temp file")
xlog.Warn("Inpainting Endpoint - failed to close orig temp file", "error", cerr)
}
origRef = origTmp.Name()
@@ -192,19 +192,19 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err
}
if cerr := maskTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close mask temp file")
xlog.Warn("Inpainting Endpoint - failed to close mask temp file", "error", cerr)
}
maskRef = maskTmp.Name()
// write JSON
enc := json.NewEncoder(jf)
if err := enc.Encode(jsonFile); err != nil {
if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file after encode error")
xlog.Warn("Inpainting Endpoint - failed to close temp json file after encode error", "error", cerr)
}
return err
}
if cerr := jf.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file")
xlog.Warn("Inpainting Endpoint - failed to close temp json file", "error", cerr)
}
// rename to desired name
if err := os.Rename(jf.Name(), jsonPath); err != nil {
@@ -216,7 +216,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
return err
}
if cerr := outTmp.Close(); cerr != nil {
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close out temp file")
xlog.Warn("Inpainting Endpoint - failed to close out temp file", "error", cerr)
}
dst = outTmp.Name() + ".png"
if err := os.Rename(outTmp.Name(), dst); err != nil {

12
core/http/endpoints/openai/mcp.go
View File

@@ -18,7 +18,7 @@ import (
"github.com/mudler/LocalAI/core/templates"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/cogito"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// MCPCompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
@@ -102,19 +102,19 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
cogito.WithContext(ctxWithCancellation),
cogito.WithMCPs(sessions...),
cogito.WithStatusCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s)
xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
}),
cogito.WithReasoningCallback(func(s string) {
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s)
xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
}),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Msgf("[model agent] [model: %s] Tool call: %s, reasoning: %s, arguments: %+v", config.Name, t.Name, t.Reasoning, t.Arguments)
xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
return cogito.ToolCallDecision{
Approved: true,
}
}),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Msgf("[model agent] [model: %s] Tool call result: %s, result: %s, tool arguments: %+v", config.Name, t.Name, t.Result, t.ToolArguments)
xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
}),
)
@@ -140,7 +140,7 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
}
jsonResult, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", jsonResult)
xlog.Debug("Response", "response", string(jsonResult))
// Return the prediction in the response body
return c.JSON(200, resp)

70
core/http/endpoints/openai/realtime.go
View File

@@ -27,7 +27,7 @@ import (
"google.golang.org/grpc"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -212,12 +212,12 @@ func registerRealtime(application *application.Application, model, intent string
return func(c *websocket.Conn) {
evaluator := application.TemplatesEvaluator()
log.Debug().Msgf("WebSocket connection established with '%s'", c.RemoteAddr().String())
xlog.Debug("WebSocket connection established", "address", c.RemoteAddr().String())
if intent != "transcription" {
sendNotImplemented(c, "Only transcription mode is supported which requires the intent=transcription parameter")
}
log.Debug().Msgf("Realtime params: model=%s, intent=%s", model, intent)
xlog.Debug("Realtime params", "model", model, "intent", intent)
sessionID := generateSessionID()
session := &Session{
@@ -265,7 +265,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ApplicationConfig(),
)
if err != nil {
log.Error().Msgf("failed to load model: %s", err.Error())
xlog.Error("failed to load model", "error", err)
sendError(c, "model_load_error", "Failed to load model", "", "")
return
}
@@ -301,14 +301,14 @@ func registerRealtime(application *application.Application, model, intent string
for {
if _, msg, err = c.ReadMessage(); err != nil {
log.Error().Msgf("read: %s", err.Error())
xlog.Error("read error", "error", err)
break
}
// Parse the incoming message
var incomingMsg IncomingMessage
if err := json.Unmarshal(msg, &incomingMsg); err != nil {
log.Error().Msgf("invalid json: %s", err.Error())
xlog.Error("invalid json", "error", err)
sendError(c, "invalid_json", "Invalid JSON format", "", "")
continue
}
@@ -316,10 +316,10 @@ func registerRealtime(application *application.Application, model, intent string
var sessionUpdate types.ClientSession
switch incomingMsg.Type {
case types.ClientEventTypeTranscriptionSessionUpdate:
log.Debug().Msgf("recv: %s", msg)
xlog.Debug("recv", "message", string(msg))
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
log.Error().Msgf("failed to unmarshal 'transcription_session.update': %s", err.Error())
xlog.Error("failed to unmarshal 'transcription_session.update'", "error", err)
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
continue
}
@@ -330,7 +330,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ModelLoader(),
application.ApplicationConfig(),
); err != nil {
log.Error().Msgf("failed to update session: %s", err.Error())
xlog.Error("failed to update session", "error", err)
sendError(c, "session_update_error", "Failed to update session", "", "")
continue
}
@@ -344,11 +344,11 @@ func registerRealtime(application *application.Application, model, intent string
})
case types.ClientEventTypeSessionUpdate:
log.Debug().Msgf("recv: %s", msg)
xlog.Debug("recv", "message", string(msg))
// Update session configurations
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
log.Error().Msgf("failed to unmarshal 'session.update': %s", err.Error())
xlog.Error("failed to unmarshal 'session.update'", "error", err)
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
continue
}
@@ -359,7 +359,7 @@ func registerRealtime(application *application.Application, model, intent string
application.ModelLoader(),
application.ApplicationConfig(),
); err != nil {
log.Error().Msgf("failed to update session: %s", err.Error())
xlog.Error("failed to update session", "error", err)
sendError(c, "session_update_error", "Failed to update session", "", "")
continue
}
@@ -373,7 +373,7 @@ func registerRealtime(application *application.Application, model, intent string
})
if session.TurnDetection.Type == types.ServerTurnDetectionTypeServerVad && !vadServerStarted {
log.Debug().Msg("Starting VAD goroutine...")
xlog.Debug("Starting VAD goroutine...")
wg.Add(1)
go func() {
defer wg.Done()
@@ -382,7 +382,7 @@ func registerRealtime(application *application.Application, model, intent string
}()
vadServerStarted = true
} else if session.TurnDetection.Type != types.ServerTurnDetectionTypeServerVad && vadServerStarted {
log.Debug().Msg("Stopping VAD goroutine...")
xlog.Debug("Stopping VAD goroutine...")
wg.Add(-1)
go func() {
@@ -393,7 +393,7 @@ func registerRealtime(application *application.Application, model, intent string
case types.ClientEventTypeInputAudioBufferAppend:
// Handle 'input_audio_buffer.append'
if incomingMsg.Audio == "" {
log.Error().Msg("Audio data is missing in 'input_audio_buffer.append'")
xlog.Error("Audio data is missing in 'input_audio_buffer.append'")
sendError(c, "missing_audio_data", "Audio data is missing", "", "")
continue
}
@@ -401,7 +401,7 @@ func registerRealtime(application *application.Application, model, intent string
// Decode base64 audio data
decodedAudio, err := base64.StdEncoding.DecodeString(incomingMsg.Audio)
if err != nil {
log.Error().Msgf("failed to decode audio data: %s", err.Error())
xlog.Error("failed to decode audio data", "error", err)
sendError(c, "invalid_audio_data", "Failed to decode audio data", "", "")
continue
}
@@ -412,7 +412,7 @@ func registerRealtime(application *application.Application, model, intent string
session.AudioBufferLock.Unlock()
case types.ClientEventTypeInputAudioBufferCommit:
log.Debug().Msgf("recv: %s", msg)
xlog.Debug("recv", "message", string(msg))
// TODO: Trigger transcription.
// TODO: Ignore this if VAD enabled or interrupt VAD?
@@ -458,12 +458,12 @@ func registerRealtime(application *application.Application, model, intent string
})
case types.ClientEventTypeConversationItemCreate:
log.Debug().Msgf("recv: %s", msg)
xlog.Debug("recv", "message", string(msg))
// Handle creating new conversation items
var item types.ConversationItemCreateEvent
if err := json.Unmarshal(incomingMsg.Item, &item); err != nil {
log.Error().Msgf("failed to unmarshal 'conversation.item.create': %s", err.Error())
xlog.Error("failed to unmarshal 'conversation.item.create'", "error", err)
sendError(c, "invalid_item", "Invalid item format", "", "")
continue
}
@@ -494,7 +494,7 @@ func registerRealtime(application *application.Application, model, intent string
var responseCreate types.ResponseCreateEvent
if len(incomingMsg.Response) > 0 {
if err := json.Unmarshal(incomingMsg.Response, &responseCreate); err != nil {
log.Error().Msgf("failed to unmarshal 'response.create' response object: %s", err.Error())
xlog.Error("failed to unmarshal 'response.create' response object", "error", err)
sendError(c, "invalid_response_create", "Invalid response create format", "", "")
continue
}
@@ -515,14 +515,14 @@ func registerRealtime(application *application.Application, model, intent string
// }()
case types.ClientEventTypeResponseCancel:
log.Printf("recv: %s", msg)
xlog.Debug("recv", "message", string(msg))
// Handle cancellation of ongoing responses
// Implement cancellation logic as needed
sendNotImplemented(c, "response.cancel")
default:
log.Error().Msgf("unknown message type: %s", incomingMsg.Type)
xlog.Error("unknown message type", "type", incomingMsg.Type)
sendError(c, "unknown_message_type", fmt.Sprintf("Unknown message type: %s", incomingMsg.Type), "", "")
}
}
@@ -542,11 +542,11 @@ func registerRealtime(application *application.Application, model, intent string
func sendEvent(c *websocket.Conn, event types.ServerEvent) {
eventBytes, err := json.Marshal(event)
if err != nil {
log.Error().Msgf("failed to marshal event: %s", err.Error())
xlog.Error("failed to marshal event", "error", err)
return
}
if err = c.WriteMessage(websocket.TextMessage, eventBytes); err != nil {
log.Error().Msgf("write: %s", err.Error())
xlog.Error("write error", "error", err)
}
}
@@ -681,10 +681,10 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
segments, err := runVAD(vadContext, session, aints)
if err != nil {
if err.Error() == "unexpected speech end" {
log.Debug().Msg("VAD cancelled")
xlog.Debug("VAD cancelled")
continue
}
log.Error().Msgf("failed to process audio: %s", err.Error())
xlog.Error("failed to process audio", "error", err)
sendError(c, "processing_error", "Failed to process audio: "+err.Error(), "", "")
continue
}
@@ -697,7 +697,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
session.AudioBufferLock.Lock()
session.InputAudioBuffer = nil
session.AudioBufferLock.Unlock()
log.Debug().Msgf("Detected silence for a while, clearing audio buffer")
xlog.Debug("Detected silence for a while, clearing audio buffer")
sendEvent(c, types.InputAudioBufferClearedEvent{
ServerEventBase: types.ServerEventBase{
@@ -729,7 +729,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
}
if float32(audioLength)-segEndTime > float32(silenceThreshold) {
log.Debug().Msgf("Detected end of speech segment")
xlog.Debug("Detected end of speech segment")
session.AudioBufferLock.Lock()
session.InputAudioBuffer = nil
session.AudioBufferLock.Unlock()
@@ -769,21 +769,21 @@ func commitUtterance(ctx context.Context, utt []byte, cfg *config.ModelConfig, e
f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav")
if err != nil {
log.Error().Msgf("failed to create temp file: %s", err.Error())
xlog.Error("failed to create temp file", "error", err)
return
}
defer f.Close()
defer os.Remove(f.Name())
log.Debug().Msgf("Writing to %s\n", f.Name())
xlog.Debug("Writing to file", "file", f.Name())
hdr := laudio.NewWAVHeader(uint32(len(utt)))
if err := hdr.Write(f); err != nil {
log.Error().Msgf("Failed to write WAV header: %s", err.Error())
xlog.Error("Failed to write WAV header", "error", err)
return
}
if _, err := f.Write(utt); err != nil {
log.Error().Msgf("Failed to write audio data: %s", err.Error())
xlog.Error("Failed to write audio data", "error", err)
return
}
@@ -1106,14 +1106,14 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
xlog.Debug("Text content to return", "text", textContentToReturn)
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
switch {
case noActionsToRun:
result, err := handleQuestion(config, input, ml, startupOptions, results, s, predInput)
if err != nil {
log.Error().Err(err).Msg("error handling question")
xlog.Error("error handling question", "error", err)
return
}
*c = append(*c, schema.Choice{
@@ -1187,7 +1187,7 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
},
}
respData, _ := json.Marshal(resp)
log.Debug().Msgf("Response: %s", respData)
xlog.Debug("Response", "response", string(respData))
// Return the prediction in the response body
return c.JSON(resp)

4
core/http/endpoints/openai/realtime_model.go
View File

@@ -9,7 +9,7 @@ import (
grpcClient "github.com/mudler/LocalAI/pkg/grpc"
"github.com/mudler/LocalAI/pkg/grpc/proto"
model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"google.golang.org/grpc"
)
@@ -209,7 +209,7 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model
}, nil
}
log.Debug().Msg("Loading a wrapped model")
xlog.Debug("Loading a wrapped model")
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
cfgLLM, err := cl.LoadModelConfigFileByName(pipeline.LLM, ml.ModelPath)

8
core/http/endpoints/openai/transcription.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
model "github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create
@@ -64,18 +64,18 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
}
if _, err := io.Copy(dstFile, f); err != nil {
log.Debug().Msgf("Audio file copying error %+v - %+v - err %+v", file.Filename, dst, err)
xlog.Debug("Audio file copying error", "filename", file.Filename, "dst", dst, "error", err)
return err
}
log.Debug().Msgf("Audio file copied to: %+v", dst)
xlog.Debug("Audio file copied", "dst", dst)
tr, err := backend.ModelTranscription(dst, input.Language, input.Translate, diarize, prompt, ml, *config, appConfig)
if err != nil {
return err
}
log.Debug().Msgf("Trascribed: %+v", tr)
xlog.Debug("Transcribed", "transcription", tr)
// TODO: handle different outputs here
return c.JSON(http.StatusOK, tr)
}

4
core/http/explorer.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/explorer"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/http/routes"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func Explorer(db *explorer.Database) *echo.Echo {
@@ -37,7 +37,7 @@ func Explorer(db *explorer.Database) *echo.Echo {
staticFS, err := fs.Sub(embedDirStatic, "static")
if err != nil {
// Log error but continue - static files might not work
log.Error().Err(err).Msg("failed to create static filesystem")
xlog.Error("failed to create static filesystem", "error", err)
} else {
e.StaticFS("/static", staticFS)
}

29
core/http/middleware/request.go
View File

@@ -17,7 +17,7 @@ import (
"github.com/mudler/LocalAI/pkg/functions"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type correlationIDKeyType string
@@ -82,7 +82,7 @@ func (re *RequestExtractor) BuildConstantDefaultModelNameMiddleware(defaultModel
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if !ok || localModelName == "" {
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, defaultModelName)
log.Debug().Str("defaultModelName", defaultModelName).Msg("context local model name not found, setting to default")
xlog.Debug("context local model name not found, setting to default", "defaultModelName", defaultModelName)
}
return next(c)
}
@@ -100,19 +100,19 @@ func (re *RequestExtractor) BuildFilteredFirstAvailableDefaultModel(filterFn con
modelNames, err := services.ListModels(re.modelConfigLoader, re.modelLoader, filterFn, services.SKIP_IF_CONFIGURED)
if err != nil {
log.Error().Err(err).Msg("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()")
xlog.Error("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()", "error", err)
return next(c)
}
if len(modelNames) == 0 {
log.Warn().Msg("SetDefaultModelNameToFirstAvailable used with no matching models installed")
xlog.Warn("SetDefaultModelNameToFirstAvailable used with no matching models installed")
// This is non-fatal - making it so was breaking the case of direct installation of raw models
// return errors.New("this endpoint requires at least one model to be installed")
return next(c)
}
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, modelNames[0])
log.Debug().Str("first model name", modelNames[0]).Msg("context local model name not found, setting to the first model")
xlog.Debug("context local model name not found, setting to the first model", "first model name", modelNames[0])
return next(c)
}
}
@@ -135,7 +135,7 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
if input.ModelName(nil) == "" {
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
if ok && localModelName != "" {
log.Debug().Str("context localModelName", localModelName).Msg("overriding empty model name in request body with value found earlier in middleware chain")
xlog.Debug("overriding empty model name in request body with value found earlier in middleware chain", "context localModelName", localModelName)
input.ModelName(&localModelName)
}
}
@@ -143,10 +143,9 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig)
if err != nil {
log.Err(err)
log.Warn().Msgf("Model Configuration File not found for %q", input.ModelName(nil))
xlog.Warn("Model Configuration File not found", "model", input.ModelName(nil), "error", err)
} else if cfg.Model == "" && input.ModelName(nil) != "" {
log.Debug().Str("input.ModelName", input.ModelName(nil)).Msg("config does not include model, using input")
xlog.Debug("config does not include model, using input", "input.ModelName", input.ModelName(nil))
cfg.Model = input.ModelName(nil)
}
@@ -203,7 +202,7 @@ func (re *RequestExtractor) SetOpenAIRequest(c echo.Context) error {
}
if cfg.Model == "" {
log.Debug().Str("input.Model", input.Model).Msg("replacing empty cfg.Model with input value")
xlog.Debug("replacing empty cfg.Model with input value", "input.Model", input.Model)
cfg.Model = input.Model
}
@@ -331,7 +330,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL)
if err != nil {
log.Error().Msgf("Failed encoding video: %s", err)
xlog.Error("Failed encoding video", "error", err)
continue CONTENT
}
input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff
@@ -341,7 +340,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL)
if err != nil {
log.Error().Msgf("Failed encoding audio: %s", err)
xlog.Error("Failed encoding audio", "error", err)
continue CONTENT
}
input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff
@@ -356,7 +355,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
// Decode content as base64 either if it's an URL or base64 text
base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL)
if err != nil {
log.Error().Msgf("Failed encoding image: %s", err)
xlog.Error("Failed encoding image", "error", err)
continue CONTENT
}
@@ -410,7 +409,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
config.TypicalP = input.TypicalP
}
log.Debug().Str("input.Input", fmt.Sprintf("%+v", input.Input))
xlog.Debug("input.Input", "input", fmt.Sprintf("%+v", input.Input))
switch inputs := input.Input.(type) {
case string:
@@ -434,7 +433,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
case string:
inputStrings = append(inputStrings, ii)
default:
log.Error().Msgf("Unknown input type: %T", ii)
xlog.Error("Unknown input type", "type", fmt.Sprintf("%T", ii))
}
}
config.InputToken = append(config.InputToken, tokens)

30
core/http/routes/ui_api.go
View File

@@ -20,7 +20,7 @@ import (
"github.com/mudler/LocalAI/core/services"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -143,11 +143,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Cancel operation endpoint
app.POST("/api/operations/:jobID/cancel", func(c echo.Context) error {
jobID := c.Param("jobID")
log.Debug().Msgf("API request to cancel operation: %s", jobID)
xlog.Debug("API request to cancel operation", "jobID", jobID)
err := galleryService.CancelOperation(jobID)
if err != nil {
log.Error().Err(err).Msgf("Failed to cancel operation: %s", jobID)
xlog.Error("Failed to cancel operation", "error", err, "jobID", jobID)
return c.JSON(http.StatusBadRequest, map[string]interface{}{
"error": err.Error(),
})
@@ -176,7 +176,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
if err != nil {
log.Error().Err(err).Msg("could not list models from galleries")
xlog.Error("could not list models from galleries", "error", err)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(),
})
@@ -246,7 +246,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Skip duplicate IDs to prevent Alpine.js x-for errors
if seenIDs[modelID] {
log.Debug().Msgf("Skipping duplicate model ID: %s", modelID)
xlog.Debug("Skipping duplicate model ID", "modelID", modelID)
continue
}
seenIDs[modelID] = true
@@ -320,7 +320,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID",
})
}
log.Debug().Msgf("API job submitted to install: %+v\n", galleryID)
xlog.Debug("API job submitted to install", "galleryID", galleryID)
id, err := uuid.NewUUID()
if err != nil {
@@ -362,7 +362,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID",
})
}
log.Debug().Msgf("API job submitted to delete: %+v\n", galleryID)
xlog.Debug("API job submitted to delete", "galleryID", galleryID)
var galleryName = galleryID
if strings.Contains(galleryID, "@") {
@@ -412,7 +412,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid model ID",
})
}
log.Debug().Msgf("API job submitted to get config for: %+v\n", galleryID)
xlog.Debug("API job submitted to get config", "galleryID", galleryID)
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
if err != nil {
@@ -498,7 +498,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
if err != nil {
log.Error().Err(err).Msg("could not list backends from galleries")
xlog.Error("could not list backends from galleries", "error", err)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(),
})
@@ -568,7 +568,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// Skip duplicate IDs to prevent Alpine.js x-for errors
if seenBackendIDs[backendID] {
log.Debug().Msgf("Skipping duplicate backend ID: %s", backendID)
xlog.Debug("Skipping duplicate backend ID", "backendID", backendID)
continue
}
seenBackendIDs[backendID] = true
@@ -640,7 +640,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend ID",
})
}
log.Debug().Msgf("API job submitted to install backend: %+v\n", backendID)
xlog.Debug("API job submitted to install backend", "backendID", backendID)
id, err := uuid.NewUUID()
if err != nil {
@@ -695,7 +695,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
})
}
log.Debug().Str("uri", req.URI).Str("name", req.Name).Str("alias", req.Alias).Msg("API job submitted to install external backend")
xlog.Debug("API job submitted to install external backend", "uri", req.URI, "name", req.Name, "alias", req.Alias)
id, err := uuid.NewUUID()
if err != nil {
@@ -745,7 +745,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend ID",
})
}
log.Debug().Msgf("API job submitted to delete backend: %+v\n", backendID)
xlog.Debug("API job submitted to delete backend", "backendID", backendID)
var backendName = backendID
if strings.Contains(backendID, "@") {
@@ -831,11 +831,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"error": "invalid backend name",
})
}
log.Debug().Msgf("API request to delete system backend: %+v\n", backendName)
xlog.Debug("API request to delete system backend", "backendName", backendName)
// Use the gallery package to delete the backend
if err := gallery.DeleteBackendFromSystem(appConfig.SystemState, backendName); err != nil {
log.Error().Err(err).Msgf("Failed to delete backend: %s", backendName)
xlog.Error("Failed to delete backend", "error", err, "backendName", backendName)
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": err.Error(),
})

12
core/p2p/federated.go
View File

@@ -5,7 +5,7 @@ import (
"math/rand/v2"
"sync"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const FederatedID = "federated"
@@ -43,7 +43,7 @@ func (fs *FederatedServer) RandomServer() string {
tunnelAddresses = append(tunnelAddresses, v.ID)
} else {
delete(fs.requestTable, v.ID) // make sure it's not tracked
log.Info().Msgf("Node %s is offline", v.ID)
xlog.Info("Node is offline", "node", v.ID)
}
}
@@ -80,7 +80,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
fs.Lock()
defer fs.Unlock()
log.Debug().Any("request_table", fs.requestTable).Msgf("SelectLeastUsedServer()")
xlog.Debug("SelectLeastUsedServer()", "request_table", fs.requestTable)
// cycle over requestTable and find the entry with the lower number
// if there are multiple entries with the same number, select one randomly
@@ -93,7 +93,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
minKey = k
}
}
log.Debug().Any("requests_served", min).Any("request_table", fs.requestTable).Msgf("Selected tunnel %s", minKey)
xlog.Debug("Selected tunnel", "tunnel", minKey, "requests_served", min, "request_table", fs.requestTable)
return minKey
}
@@ -104,7 +104,7 @@ func (fs *FederatedServer) RecordRequest(nodeID string) {
// increment the counter for the nodeID in the requestTable
fs.requestTable[nodeID]++
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Recording request")
xlog.Debug("Recording request", "request_table", fs.requestTable, "request", nodeID)
}
func (fs *FederatedServer) ensureRecordExist(nodeID string) {
@@ -114,5 +114,5 @@ func (fs *FederatedServer) ensureRecordExist(nodeID string) {
fs.requestTable[nodeID] = 0
}
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Ensure record exists")
xlog.Debug("Ensure record exists", "request_table", fs.requestTable, "request", nodeID)
}

22
core/p2p/federated_server.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/edgevpn/pkg/node"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func (f *FederatedServer) Start(ctx context.Context) error {
@@ -23,7 +23,7 @@ func (f *FederatedServer) Start(ctx context.Context) error {
}
if err := ServiceDiscoverer(ctx, n, f.p2ptoken, f.service, func(servicesID string, tunnel schema.NodeData) {
log.Debug().Msgf("Discovered node: %s", tunnel.ID)
xlog.Debug("Discovered node", "node", tunnel.ID)
}, false); err != nil {
return err
}
@@ -33,11 +33,11 @@ func (f *FederatedServer) Start(ctx context.Context) error {
func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
log.Info().Msgf("Allocating service '%s' on: %s", fs.service, fs.listenAddr)
xlog.Info("Allocating service", "service", fs.service, "address", fs.listenAddr)
// Open local port for listening
l, err := net.Listen("tcp", fs.listenAddr)
if err != nil {
log.Error().Err(err).Msg("Error listening")
xlog.Error("Error listening", "error", err)
return err
}
@@ -54,7 +54,7 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
case <-ctx.Done():
return errors.New("context canceled")
default:
log.Debug().Msgf("New connection from %s", l.Addr().String())
xlog.Debug("New connection", "address", l.Addr().String())
// Listen for an incoming connection.
conn, err := l.Accept()
if err != nil {
@@ -68,11 +68,11 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
if fs.workerTarget != "" {
workerID = fs.workerTarget
} else if fs.loadBalanced {
log.Debug().Msgf("Load balancing request")
xlog.Debug("Load balancing request")
workerID = fs.SelectLeastUsedServer()
if workerID == "" {
log.Debug().Msgf("Least used server not found, selecting random")
xlog.Debug("Least used server not found, selecting random")
workerID = fs.RandomServer()
}
} else {
@@ -80,15 +80,15 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
}
if workerID == "" {
log.Error().Msg("No available nodes yet")
xlog.Error("No available nodes yet")
fs.sendHTMLResponse(conn, 503, "Sorry, waiting for nodes to connect")
return
}
log.Debug().Msgf("Selected node %s", workerID)
xlog.Debug("Selected node", "node", workerID)
nodeData, exists := GetNode(fs.service, workerID)
if !exists {
log.Error().Msgf("Node %s not found", workerID)
xlog.Error("Node not found", "node", workerID)
fs.sendHTMLResponse(conn, 404, "Node not found")
return
}
@@ -123,7 +123,7 @@ func (fs *FederatedServer) sendHTMLResponse(conn net.Conn, statusCode int, messa
// Write the response to the client connection.
_, writeErr := io.WriteString(conn, response)
if writeErr != nil {
log.Error().Err(writeErr).Msg("Error writing response to client")
xlog.Error("Error writing response to client", "error", writeErr)
}
}

30
core/p2p/p2p.go
View File

@@ -21,9 +21,9 @@ import (
"github.com/mudler/edgevpn/pkg/services"
"github.com/mudler/edgevpn/pkg/types"
eutils "github.com/mudler/edgevpn/pkg/utils"
zlog "github.com/mudler/xlog"
"github.com/multiformats/go-multiaddr"
"github.com/phayes/freeport"
zlog "github.com/rs/zerolog/log"
"github.com/mudler/edgevpn/pkg/logger"
)
@@ -94,7 +94,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
existingValue.Unmarshal(service)
// If mismatch, update the blockchain
if !found {
zlog.Error().Msg("Service not found on blockchain")
zlog.Error("Service not found on blockchain")
conn.Close()
// ll.Debugf("service '%s' not found on blockchain", serviceID)
return
@@ -103,7 +103,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
// Decode the Peer
d, err := peer.Decode(service.PeerID)
if err != nil {
zlog.Error().Msg("cannot decode peer")
zlog.Error("cannot decode peer")
conn.Close()
// ll.Debugf("could not decode peer '%s'", service.PeerID)
@@ -113,14 +113,14 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
// Open a stream
stream, err := node.Host().NewStream(ctx, d, protocol.ServiceProtocol.ID())
if err != nil {
zlog.Error().Err(err).Msg("cannot open stream peer")
zlog.Error("cannot open stream peer", "error", err)
conn.Close()
// ll.Debugf("could not open stream '%s'", err.Error())
return
}
// ll.Debugf("(service %s) Redirecting", serviceID, l.Addr().String())
zlog.Info().Msgf("Redirecting %s to %s", conn.LocalAddr().String(), stream.Conn().RemoteMultiaddr().String())
zlog.Info("Redirecting", "from", conn.LocalAddr().String(), "to", stream.Conn().RemoteMultiaddr().String())
closer := make(chan struct{}, 2)
go copyStream(closer, stream, conn)
go copyStream(closer, conn, stream)
@@ -131,11 +131,11 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
}
func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error {
zlog.Info().Msgf("Allocating service '%s' on: %s", service, listenAddr)
zlog.Info("Allocating service", "service", service, "address", listenAddr)
// Open local port for listening
l, err := net.Listen("tcp", listenAddr)
if err != nil {
zlog.Error().Err(err).Msg("Error listening")
zlog.Error("Error listening", "error", err)
return err
}
go func() {
@@ -151,7 +151,7 @@ func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, serv
case <-ctx.Done():
return errors.New("context canceled")
default:
zlog.Debug().Msg("New for connection")
zlog.Debug("New for connection")
// Listen for an incoming connection.
conn, err := l.Accept()
if err != nil {
@@ -187,7 +187,7 @@ func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID stri
for {
select {
case <-ctx.Done():
zlog.Error().Msg("Discoverer stopped")
zlog.Error("Discoverer stopped")
return
case tunnel := <-tunnels:
AddNode(servicesID, tunnel)
@@ -220,7 +220,7 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
for {
select {
case <-ctx.Done():
zlog.Error().Msg("Discoverer stopped")
zlog.Error("Discoverer stopped")
return
default:
time.Sleep(5 * time.Second)
@@ -230,14 +230,14 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
if logLevel == logLevelDebug {
// We want to surface this debugging data only if p2p logging is set to debug
// (and not generally the whole application, as this can be really noisy)
zlog.Debug().Any("data", ledger.LastBlock().Storage).Msg("Ledger data")
zlog.Debug("Ledger data", "data", ledger.LastBlock().Storage)
}
for k, v := range data {
// New worker found in the ledger data as k (worker id)
nd := &schema.NodeData{}
if err := v.Unmarshal(nd); err != nil {
zlog.Error().Msg("cannot unmarshal node data")
zlog.Error("cannot unmarshal node data")
continue
}
ensureService(ctx, n, nd, k, allocate)
@@ -278,7 +278,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
// Start the service
port, err := freeport.GetFreePort()
if err != nil {
zlog.Error().Err(err).Msgf("Could not allocate a free port for %s", nd.ID)
zlog.Error("Could not allocate a free port", "error", err, "node", nd.ID)
cancel()
return
}
@@ -286,7 +286,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
nd.TunnelAddress = tunnelAddress
go allocateLocalService(newCtxm, n, tunnelAddress, sserv)
zlog.Debug().Msgf("Starting service %s on %s", sserv, tunnelAddress)
zlog.Debug("Starting service", "service", sserv, "address", tunnelAddress)
}
service[nd.Name] = nodeServiceData{
NodeData: *nd,
@@ -298,7 +298,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
if !nd.IsOnline() && !ndService.NodeData.IsOnline() {
ndService.CancelFunc()
delete(service, nd.Name)
zlog.Info().Msgf("Node %s is offline, deleting", nd.ID)
zlog.Info("Node is offline, deleting", "node", nd.ID)
} else if nd.IsOnline() {
// update last seen inside service
nd.TunnelAddress = ndService.NodeData.TunnelAddress

4
core/schema/message.go
View File

@@ -3,7 +3,7 @@ package schema
import (
"encoding/json"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/pkg/grpc/proto"
)
@@ -72,7 +72,7 @@ func (messages Messages) ToProto() []*proto.Message {
if len(message.ToolCalls) > 0 {
toolCallsJSON, err := json.Marshal(message.ToolCalls)
if err != nil {
log.Warn().Err(err).Msg("failed to marshal tool_calls to JSON")
xlog.Warn("failed to marshal tool_calls to JSON", "error", err)
} else {
protoMessages[i].ToolCalls = string(toolCallsJSON)
}

94
core/services/agent_jobs.go
View File

@@ -28,7 +28,7 @@ import (
"github.com/mudler/LocalAI/pkg/xsync"
"github.com/mudler/cogito"
"github.com/robfig/cron/v3"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// AgentJobService manages agent tasks and job execution
@@ -126,7 +126,7 @@ func (s *AgentJobService) LoadTasksFromFile() error {
defer s.fileMutex.Unlock()
if _, err := os.Stat(s.tasksFile); os.IsNotExist(err) {
log.Debug().Msg("agent_tasks.json not found, starting with empty tasks")
xlog.Debug("agent_tasks.json not found, starting with empty tasks")
return nil
}
@@ -145,12 +145,12 @@ func (s *AgentJobService) LoadTasksFromFile() error {
// Schedule cron if enabled and has cron expression
if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", task.ID).Msg("Failed to schedule cron task on load")
xlog.Warn("Failed to schedule cron task on load", "error", err, "task_id", task.ID)
}
}
}
log.Info().Int("count", len(tasksFile.Tasks)).Msg("Loaded tasks from file")
xlog.Info("Loaded tasks from file", "count", len(tasksFile.Tasks))
return nil
}
@@ -190,7 +190,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
defer s.fileMutex.Unlock()
if _, err := os.Stat(s.jobsFile); os.IsNotExist(err) {
log.Debug().Msg("agent_jobs.json not found, starting with empty jobs")
xlog.Debug("agent_jobs.json not found, starting with empty jobs")
return nil
}
@@ -209,7 +209,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
s.jobs.Set(job.ID, job)
}
log.Info().Int("count", len(jobsFile.Jobs)).Msg("Loaded jobs from file")
xlog.Info("Loaded jobs from file", "count", len(jobsFile.Jobs))
return nil
}
@@ -267,14 +267,14 @@ func (s *AgentJobService) CreateTask(task schema.Task) (string, error) {
// Schedule cron if enabled and has cron expression
if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task")
xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
// Don't fail task creation if cron scheduling fails
}
}
// Save to file
if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file")
xlog.Error("Failed to save tasks to file", "error", err)
// Don't fail task creation if file save fails
}
@@ -304,13 +304,13 @@ func (s *AgentJobService) UpdateTask(id string, task schema.Task) error {
// Schedule new cron if enabled and has cron expression
if task.Enabled && task.Cron != "" {
if err := s.ScheduleCronTask(task); err != nil {
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task")
xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
}
}
// Save to file
if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file")
xlog.Error("Failed to save tasks to file", "error", err)
}
return nil
@@ -330,7 +330,7 @@ func (s *AgentJobService) DeleteTask(id string) error {
// Save to file
if err := s.SaveTasksToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save tasks to file")
xlog.Error("Failed to save tasks to file", "error", err)
}
return nil
@@ -409,7 +409,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
// Fetch content from URL with custom headers
dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type)
if err != nil {
log.Warn().Err(err).Str("url", source.URL).Str("type", source.Type).Msg("Failed to fetch multimedia from task source")
xlog.Warn("Failed to fetch multimedia from task source", "error", err, "url", source.URL, "type", source.Type)
continue
}
@@ -449,7 +449,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
// Save to file (async, don't block)
go func() {
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file")
xlog.Error("Failed to save jobs to file", "error", err)
}
}()
@@ -544,7 +544,7 @@ func (s *AgentJobService) CancelJob(id string) error {
// Save to file (async)
go func() {
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file")
xlog.Error("Failed to save jobs to file", "error", err)
}
}()
@@ -561,7 +561,7 @@ func (s *AgentJobService) DeleteJob(id string) error {
// Save to file
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file")
xlog.Error("Failed to save jobs to file", "error", err)
}
return nil
@@ -750,7 +750,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Images) > 0 {
images, err := s.convertToMultimediaContent(job.Images, JobImageType)
if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert images")
xlog.Warn("Failed to convert images", "error", err, "job_id", job.ID)
} else {
multimediaItems = append(multimediaItems, images...)
}
@@ -760,7 +760,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Videos) > 0 {
videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType)
if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert videos")
xlog.Warn("Failed to convert videos", "error", err, "job_id", job.ID)
} else {
multimediaItems = append(multimediaItems, videos...)
}
@@ -770,7 +770,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Audios) > 0 {
audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType)
if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert audios")
xlog.Warn("Failed to convert audios", "error", err, "job_id", job.ID)
} else {
multimediaItems = append(multimediaItems, audios...)
}
@@ -780,7 +780,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
if len(job.Files) > 0 {
files, err := s.convertToMultimediaContent(job.Files, JobFileType)
if err != nil {
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert files")
xlog.Warn("Failed to convert files", "error", err, "job_id", job.ID)
} else {
multimediaItems = append(multimediaItems, files...)
}
@@ -817,7 +817,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
cogito.WithContext(ctx),
cogito.WithMCPs(sessions...),
cogito.WithStatusCallback(func(status string) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Status: %s", status)
xlog.Debug("Status", "job_id", job.ID, "model", modelConfig.Name, "status", status)
// Store trace
trace := schema.JobTrace{
Type: "status",
@@ -828,7 +828,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
s.jobs.Set(job.ID, job)
}),
cogito.WithReasoningCallback(func(reasoning string) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Reasoning: %s", reasoning)
xlog.Debug("Reasoning", "job_id", job.ID, "model", modelConfig.Name, "reasoning", reasoning)
// Store trace
trace := schema.JobTrace{
Type: "reasoning",
@@ -839,9 +839,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
s.jobs.Set(job.ID, job)
}),
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).
Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).
Msg("Tool call")
xlog.Debug("Tool call", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
// Store trace
arguments := make(map[string]interface{})
if t.Arguments != nil {
@@ -861,9 +859,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
}
}),
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).
Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).
Msg("Tool call result")
xlog.Debug("Tool call result", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
// Store trace
arguments := make(map[string]interface{})
// Convert ToolArguments to map via JSON marshaling
@@ -988,7 +984,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
// Save to file (async)
go func() {
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file")
xlog.Error("Failed to save jobs to file", "error", err)
}
}()
@@ -1023,7 +1019,7 @@ func (s *AgentJobService) worker(ctx context.Context) {
// Execute job
err := s.executeJobInternal(exec.Job, exec.Task, exec.Ctx)
if err != nil {
log.Error().Err(err).Str("job_id", exec.Job.ID).Msg("Job execution failed")
xlog.Error("Job execution failed", "error", err, "job_id", exec.Job.ID)
}
// Clean up cancellation
@@ -1051,7 +1047,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
// Multimedia will be fetched from task sources in ExecuteJob
_, err := s.ExecuteJob(task.ID, cronParams, "cron", nil)
if err != nil {
log.Error().Err(err).Str("task_id", task.ID).Msg("Failed to execute cron job")
xlog.Error("Failed to execute cron job", "error", err, "task_id", task.ID)
}
})
if err != nil {
@@ -1059,7 +1055,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
}
s.cronEntries.Set(task.ID, entryID)
log.Info().Str("task_id", task.ID).Str("cron", cronExpr).Msg("Scheduled cron task")
xlog.Info("Scheduled cron task", "task_id", task.ID, "cron", cronExpr)
return nil
}
@@ -1069,7 +1065,7 @@ func (s *AgentJobService) UnscheduleCronTask(taskID string) {
entryID := s.cronEntries.Get(taskID)
s.cronScheduler.Remove(entryID)
s.cronEntries.Delete(taskID)
log.Info().Str("task_id", taskID).Msg("Unscheduled cron task")
xlog.Info("Unscheduled cron task", "task_id", taskID)
}
}
@@ -1082,7 +1078,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
return // No webhooks configured
}
log.Info().Str("job_id", job.ID).Int("webhook_count", len(webhookConfigs)).Msg("Sending webhooks")
xlog.Info("Sending webhooks", "job_id", job.ID, "webhook_count", len(webhookConfigs))
// Send all webhooks concurrently and track results
var wg sync.WaitGroup
@@ -1140,7 +1136,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
// Save to file (async)
go func() {
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file")
xlog.Error("Failed to save jobs to file", "error", err)
}
}()
}
@@ -1157,11 +1153,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
// Build payload
payload, err := s.buildWebhookPayload(job, task, webhookConfig)
if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to build webhook payload")
xlog.Error("Failed to build webhook payload", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("failed to build payload: %w", err)
}
log.Debug().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Str("payload", string(payload)).Msg("Sending webhook")
xlog.Debug("Sending webhook", "job_id", job.ID, "webhook_url", webhookConfig.URL, "payload", string(payload))
// Determine HTTP method (default to POST)
method := webhookConfig.Method
@@ -1172,7 +1168,7 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
// Create HTTP request
req, err := http.NewRequest(method, webhookConfig.URL, bytes.NewBuffer(payload))
if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to create webhook request")
xlog.Error("Failed to create webhook request", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("failed to create request: %w", err)
}
@@ -1186,11 +1182,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
client := &http.Client{Timeout: 30 * time.Second}
err = s.executeWithRetry(client, req)
if err != nil {
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivery failed")
xlog.Error("Webhook delivery failed", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
return fmt.Errorf("webhook delivery failed: %w", err)
}
log.Info().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivered successfully")
xlog.Info("Webhook delivered successfully", "job_id", job.ID, "webhook_url", webhookConfig.URL)
return nil
}
@@ -1303,10 +1299,10 @@ func (s *AgentJobService) CleanupOldJobs() error {
}
if removed > 0 {
log.Info().Int("removed", removed).Int("retention_days", s.retentionDays).Msg("Cleaned up old jobs")
xlog.Info("Cleaned up old jobs", "removed", removed, "retention_days", s.retentionDays)
// Save to file
if err := s.SaveJobsToFile(); err != nil {
log.Error().Err(err).Msg("Failed to save jobs to file after cleanup")
xlog.Error("Failed to save jobs to file after cleanup", "error", err)
}
}
@@ -1327,10 +1323,10 @@ func (s *AgentJobService) Start(ctx context.Context) error {
// Load tasks and jobs from files
if err := s.LoadTasksFromFile(); err != nil {
log.Warn().Err(err).Msg("Failed to load tasks from file")
xlog.Warn("Failed to load tasks from file", "error", err)
}
if err := s.LoadJobsFromFile(); err != nil {
log.Warn().Err(err).Msg("Failed to load jobs from file")
xlog.Warn("Failed to load jobs from file", "error", err)
}
// Start cron scheduler
@@ -1345,19 +1341,19 @@ func (s *AgentJobService) Start(ctx context.Context) error {
// Schedule daily cleanup at midnight
_, err := s.cronScheduler.AddFunc("0 0 * * *", func() {
if err := s.CleanupOldJobs(); err != nil {
log.Error().Err(err).Msg("Failed to cleanup old jobs")
xlog.Error("Failed to cleanup old jobs", "error", err)
}
})
if err != nil {
log.Warn().Err(err).Msg("Failed to schedule daily cleanup")
xlog.Warn("Failed to schedule daily cleanup", "error", err)
}
// Run initial cleanup
if err := s.CleanupOldJobs(); err != nil {
log.Warn().Err(err).Msg("Failed to run initial cleanup")
xlog.Warn("Failed to run initial cleanup", "error", err)
}
log.Info().Int("retention_days", s.retentionDays).Msg("AgentJobService started")
xlog.Info("AgentJobService started", "retention_days", s.retentionDays)
return nil
}
@@ -1370,7 +1366,7 @@ func (s *AgentJobService) Stop() error {
if s.cronScheduler != nil {
s.cronScheduler.Stop()
}
log.Info().Msg("AgentJobService stopped")
xlog.Info("AgentJobService stopped")
return nil
}
@@ -1380,5 +1376,5 @@ func (s *AgentJobService) UpdateRetentionDays(days int) {
if days == 0 {
s.retentionDays = 30 // Default
}
log.Info().Int("retention_days", s.retentionDays).Msg("Updated agent job retention days")
xlog.Info("Updated agent job retention days", "retention_days", s.retentionDays)
}

230
core/services/backend_monitor.go
View File

@@ -1,115 +1,115 @@
package services
import (
"context"
"fmt"
"strings"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/rs/zerolog/log"
gopsutil "github.com/shirou/gopsutil/v3/process"
)
type BackendMonitorService struct {
modelConfigLoader *config.ModelConfigLoader
modelLoader *model.ModelLoader
options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name.
}
func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService {
return &BackendMonitorService{
modelLoader: modelLoader,
modelConfigLoader: configLoader,
options: appConfig,
}
}
func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*schema.BackendMonitorResponse, error) {
config, exists := bms.modelConfigLoader.GetModelConfig(model)
var backend string
if exists {
backend = config.Model
} else {
// Last ditch effort: use it raw, see if a backend happens to match.
backend = model
}
if !strings.HasSuffix(backend, ".bin") {
backend = fmt.Sprintf("%s.bin", backend)
}
pid, err := bms.modelLoader.GetGRPCPID(backend)
if err != nil {
log.Error().Err(err).Str("model", model).Msg("failed to find GRPC pid")
return nil, err
}
// Name is slightly frightening but this does _not_ create a new process, rather it looks up an existing process by PID.
backendProcess, err := gopsutil.NewProcess(int32(pid))
if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting process info")
return nil, err
}
memInfo, err := backendProcess.MemoryInfo()
if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory info")
return nil, err
}
memPercent, err := backendProcess.MemoryPercent()
if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory percent")
return nil, err
}
cpuPercent, err := backendProcess.CPUPercent()
if err != nil {
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting cpu percent")
return nil, err
}
return &schema.BackendMonitorResponse{
MemoryInfo: memInfo,
MemoryPercent: memPercent,
CPUPercent: cpuPercent,
}, nil
}
func (bms BackendMonitorService) CheckAndSample(modelName string) (*proto.StatusResponse, error) {
modelAddr := bms.modelLoader.CheckIsLoaded(modelName)
if modelAddr == nil {
return nil, fmt.Errorf("backend %s is not currently loaded", modelName)
}
status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO())
if rpcErr != nil {
log.Warn().Msgf("backend %s experienced an error retrieving status info: %s", modelName, rpcErr.Error())
val, slbErr := bms.SampleLocalBackendProcess(modelName)
if slbErr != nil {
return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error())
}
return &proto.StatusResponse{
State: proto.StatusResponse_ERROR,
Memory: &proto.MemoryUsageData{
Total: val.MemoryInfo.VMS,
Breakdown: map[string]uint64{
"gopsutil-RSS": val.MemoryInfo.RSS,
},
},
}, nil
}
return status, nil
}
func (bms BackendMonitorService) ShutdownModel(modelName string) error {
return bms.modelLoader.ShutdownModel(modelName)
}
package services
import (
"context"
"fmt"
"strings"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/xlog"
gopsutil "github.com/shirou/gopsutil/v3/process"
)
type BackendMonitorService struct {
modelConfigLoader *config.ModelConfigLoader
modelLoader *model.ModelLoader
options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name.
}
func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService {
return &BackendMonitorService{
modelLoader: modelLoader,
modelConfigLoader: configLoader,
options: appConfig,
}
}
func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*schema.BackendMonitorResponse, error) {
config, exists := bms.modelConfigLoader.GetModelConfig(model)
var backend string
if exists {
backend = config.Model
} else {
// Last ditch effort: use it raw, see if a backend happens to match.
backend = model
}
if !strings.HasSuffix(backend, ".bin") {
backend = fmt.Sprintf("%s.bin", backend)
}
pid, err := bms.modelLoader.GetGRPCPID(backend)
if err != nil {
xlog.Error("failed to find GRPC pid", "error", err, "model", model)
return nil, err
}
// Name is slightly frightening but this does _not_ create a new process, rather it looks up an existing process by PID.
backendProcess, err := gopsutil.NewProcess(int32(pid))
if err != nil {
xlog.Error("error getting process info", "error", err, "model", model, "pid", pid)
return nil, err
}
memInfo, err := backendProcess.MemoryInfo()
if err != nil {
xlog.Error("error getting memory info", "error", err, "model", model, "pid", pid)
return nil, err
}
memPercent, err := backendProcess.MemoryPercent()
if err != nil {
xlog.Error("error getting memory percent", "error", err, "model", model, "pid", pid)
return nil, err
}
cpuPercent, err := backendProcess.CPUPercent()
if err != nil {
xlog.Error("error getting cpu percent", "error", err, "model", model, "pid", pid)
return nil, err
}
return &schema.BackendMonitorResponse{
MemoryInfo: memInfo,
MemoryPercent: memPercent,
CPUPercent: cpuPercent,
}, nil
}
func (bms BackendMonitorService) CheckAndSample(modelName string) (*proto.StatusResponse, error) {
modelAddr := bms.modelLoader.CheckIsLoaded(modelName)
if modelAddr == nil {
return nil, fmt.Errorf("backend %s is not currently loaded", modelName)
}
status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO())
if rpcErr != nil {
xlog.Warn("backend experienced an error retrieving status info", "backend", modelName, "error", rpcErr)
val, slbErr := bms.SampleLocalBackendProcess(modelName)
if slbErr != nil {
return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error())
}
return &proto.StatusResponse{
State: proto.StatusResponse_ERROR,
Memory: &proto.MemoryUsageData{
Total: val.MemoryInfo.VMS,
Breakdown: map[string]uint64{
"gopsutil-RSS": val.MemoryInfo.RSS,
},
},
}, nil
}
return status, nil
}
func (bms BackendMonitorService) ShutdownModel(modelName string) error {
return bms.modelLoader.ShutdownModel(modelName)
}

16
core/services/backends.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, any], systemState *system.SystemState) error {
@@ -62,7 +62,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
g.modelLoader.DeleteExternalBackend(op.GalleryElementName)
} else if op.ExternalURI != "" {
// External backend installation (OCI image, URL, or path)
log.Info().Str("uri", op.ExternalURI).Str("name", op.ExternalName).Str("alias", op.ExternalAlias).Msg("Installing external backend")
xlog.Info("Installing external backend", "uri", op.ExternalURI, "name", op.ExternalName, "alias", op.ExternalAlias)
err = InstallExternalBackend(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, progressCallback, op.ExternalURI, op.ExternalName, op.ExternalAlias)
// Update GalleryElementName for status tracking if a name was derived
if op.ExternalName != "" {
@@ -70,8 +70,8 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
}
} else {
// Standard gallery installation
log.Warn().Msgf("installing backend %s", op.GalleryElementName)
log.Debug().Msgf("backend galleries: %v", g.appConfig.BackendGalleries)
xlog.Warn("installing backend", "backend", op.GalleryElementName)
xlog.Debug("backend galleries", "galleries", g.appConfig.BackendGalleries)
err = gallery.InstallBackendFromGallery(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true)
}
if err != nil {
@@ -85,7 +85,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
})
return err
}
log.Error().Err(err).Msgf("error installing backend %s", op.GalleryElementName)
xlog.Error("error installing backend", "error", err, "backend", op.GalleryElementName)
if !op.Delete {
// If we didn't install the backend, we need to make sure we don't have a leftover directory
gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName)
@@ -114,7 +114,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
if name == "" { // infer it from the path
name = filepath.Base(backend)
}
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from path")
xlog.Info("Installing backend from path", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{
Name: name,
@@ -128,7 +128,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
if name == "" {
return fmt.Errorf("specifying a name is required for OCI images")
}
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image")
xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{
Name: name,
@@ -150,7 +150,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
name = derivedName
}
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image")
xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
Metadata: gallery.Metadata{
Name: name,

4
core/services/metrics.go
View File

@@ -3,7 +3,7 @@ package services
import (
"context"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/metric"
@@ -49,6 +49,6 @@ func (lams LocalAIMetricsService) Shutdown() error {
//// setupOTelSDK bootstraps the OpenTelemetry pipeline.
//// If it does not return an error, make sure to call shutdown for proper cleanup.
log.Warn().Msgf("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?")
xlog.Warn("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?")
return nil
}

4
core/services/models.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"gopkg.in/yaml.v2"
)
@@ -208,7 +208,7 @@ func processModelOperation(
return err
}
if automaticallyInstallBackend && installedModel.Backend != "" {
log.Debug().Msgf("Installing backend %q", installedModel.Backend)
xlog.Debug("Installing backend", "backend", installedModel.Backend)
if err := gallery.InstallBackendFromGallery(ctx, op.BackendGalleries, systemState, modelLoader, installedModel.Backend, progressCallback, false); err != nil {
return err
}

14
core/startup/model_preload.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -32,10 +32,10 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
// Check if it's a model gallery, or print a warning
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries)
if e != nil && found {
log.Error().Err(err).Msgf("[startup] failed installing model '%s'", url)
xlog.Error("[startup] failed installing model", "error", err, "model", url)
err = errors.Join(err, e)
} else if !found {
log.Debug().Msgf("[startup] model not found in the gallery '%s'", url)
xlog.Debug("[startup] model not found in the gallery", "model", url)
if galleryService == nil {
return fmt.Errorf("cannot start autoimporter, not sure how to handle this uri")
@@ -44,7 +44,7 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
// TODO: we should just use the discoverModelConfig here and default to this.
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
if discoverErr != nil {
log.Error().Err(discoverErr).Msgf("[startup] failed to discover model config '%s'", url)
xlog.Error("[startup] failed to discover model config", "error", discoverErr, "model", url)
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
continue
}
@@ -76,11 +76,11 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
}
if status.Error != nil {
log.Error().Err(status.Error).Msgf("[startup] failed to import model '%s' from '%s'", modelConfig.Name, url)
xlog.Error("[startup] failed to import model", "error", status.Error, "model", modelConfig.Name, "url", url)
return status.Error
}
log.Info().Msgf("[startup] imported model '%s' from '%s'", modelConfig.Name, url)
xlog.Info("[startup] imported model", "model", modelConfig.Name, "url", url)
}
}
return err
@@ -101,7 +101,7 @@ func installModel(ctx context.Context, galleries, backendGalleries []config.Gall
downloadStatus = utils.DisplayDownloadFunction
}
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model")
xlog.Info("installing model", "model", modelName, "license", model.License)
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries)
if err != nil {
return err, true

14
core/templates/evaluator.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/functions"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// Rather than pass an interface{} to the prompt template:
@@ -133,13 +133,13 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
}
templatedChatMessage, err := e.evaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData)
if err != nil {
log.Error().Err(err).Interface("message", chatMessageData).Str("template", config.TemplateConfig.ChatMessage).Msg("error processing message with template, skipping")
xlog.Error("error processing message with template, skipping", "error", err, "message", chatMessageData, "template", config.TemplateConfig.ChatMessage)
} else {
if templatedChatMessage == "" {
log.Warn().Msgf("template \"%s\" produced blank output for %+v. Skipping!", config.TemplateConfig.ChatMessage, chatMessageData)
xlog.Warn("template produced blank output, skipping", "template", config.TemplateConfig.ChatMessage, "message", chatMessageData)
continue // TODO: This continue is here intentionally to skip over the line `mess = append(mess, content)` below, and to prevent the sprintf
}
log.Debug().Msgf("templated message for chat: %s", templatedChatMessage)
xlog.Debug("templated message for chat", "message", templatedChatMessage)
content = templatedChatMessage
}
}
@@ -203,7 +203,7 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
}
predInput = strings.Join(mess, joinCharacter)
log.Debug().Msgf("Prompt (before templating): %s", predInput)
xlog.Debug("Prompt (before templating)", "prompt", predInput)
promptTemplate := ChatPromptTemplate
@@ -221,9 +221,9 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
})
if err == nil {
predInput = templatedInput
log.Debug().Msgf("Template found, input modified to: %s", predInput)
xlog.Debug("Template found, input modified", "input", predInput)
} else {
log.Debug().Msgf("Template failed loading: %s", err.Error())
xlog.Debug("Template failed loading", "error", err)
}
return predInput

2
go.mod
View File

@@ -41,7 +41,7 @@ require (
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2
github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.34.0
github.com/mudler/xlog v0.0.2
github.com/russross/blackfriday v1.6.0
github.com/sashabaranov/go-openai v1.41.2
github.com/schollz/progressbar/v3 v3.18.0

2
go.sum
View File

@@ -518,6 +518,8 @@ github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2 h1:+WHsL/j6EWOMUiMVI
github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2/go.mod h1:EA8Ashhd56o32qN7ouPKFSRUs/Z+LrRCF4v6R2Oarm8=
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025 h1:WFLP5FHInarYGXi6B/Ze204x7Xy6q/I4nCZnWEyPHK0=
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025/go.mod h1:QuIFdRstyGJt+MTTkWY+mtD7U6xwjOR6SwKUjmLZtR4=
github.com/mudler/xlog v0.0.2 h1:w3Z3HIexjYduveotEMlHAeKjTFDi15NzZLBtfV76i1Y=
github.com/mudler/xlog v0.0.2/go.mod h1:39f5vcd05Qd6GWKM8IjyHNQ7AmOx3ZM0YfhfIGhC18U=
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=

28
pkg/downloader/uri.go
View File

@@ -20,7 +20,7 @@ import (
"github.com/mudler/LocalAI/pkg/oci"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/LocalAI/pkg/xio"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -70,7 +70,7 @@ func (uri URI) ReadWithAuthorizationAndCallback(ctx context.Context, basePath st
// Check if the local file is rooted in basePath
err = utils.InTrustedRoot(resolvedFile, resolvedBasePath)
if err != nil {
log.Debug().Str("resolvedFile", resolvedFile).Str("basePath", basePath).Msg("downloader.GetURI blocked an attempt to ready a file url outside of basePath")
xlog.Debug("downloader.GetURI blocked an attempt to ready a file url outside of basePath", "resolvedFile", resolvedFile, "basePath", basePath)
return err
}
// Read the response body
@@ -245,11 +245,11 @@ func (s URI) ResolveURL() string {
func removePartialFile(tmpFilePath string) error {
_, err := os.Stat(tmpFilePath)
if err == nil {
log.Debug().Msgf("Removing temporary file %s", tmpFilePath)
xlog.Debug("Removing temporary file", "file", tmpFilePath)
err = os.Remove(tmpFilePath)
if err != nil {
err1 := fmt.Errorf("failed to remove temporary download file %s: %v", tmpFilePath, err)
log.Warn().Msg(err1.Error())
xlog.Warn("failed to remove temporary download file", "error", err1)
return err1
}
}
@@ -333,7 +333,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// Check if the file already exists
_, err := os.Stat(filePath)
if err == nil {
log.Debug().Str("filePath", filePath).Msg("[downloader] File already exists")
xlog.Debug("[downloader] File already exists", "filePath", filePath)
// File exists, check SHA
if sha != "" {
// Verify SHA
@@ -343,7 +343,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
}
if calculatedSHA == sha {
// SHA matches, skip downloading
log.Debug().Msgf("File %q already exists and matches the SHA. Skipping download", filePath)
xlog.Debug("File already exists and matches the SHA. Skipping download", "file", filePath)
return nil
}
// SHA doesn't match, delete the file and download again
@@ -351,11 +351,11 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
if err != nil {
return fmt.Errorf("failed to remove existing file %q: %v", filePath, err)
}
log.Debug().Msgf("Removed %q (SHA doesn't match)", filePath)
xlog.Debug("Removed file (SHA doesn't match)", "file", filePath)
} else {
// SHA is missing, skip downloading
log.Debug().Msgf("File %q already exists. Skipping download", filePath)
xlog.Debug("File already exists. Skipping download", "file", filePath)
return nil
}
} else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() {
@@ -363,7 +363,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url)
}
log.Info().Msgf("Downloading %s", url)
xlog.Info("Downloading", "url", url)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
@@ -480,19 +480,19 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// Verify SHA
calculatedSHA := fmt.Sprintf("%x", progress.hash.Sum(nil))
if calculatedSHA != sha {
log.Debug().Msgf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
xlog.Debug("SHA mismatch for file", "file", filePath, "calculated", calculatedSHA, "metadata", sha)
return fmt.Errorf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
}
} else {
log.Debug().Msgf("SHA missing for %q. Skipping validation", filePath)
xlog.Debug("SHA missing. Skipping validation", "file", filePath)
}
log.Info().Msgf("File %q downloaded and verified", filePath)
xlog.Info("File downloaded and verified", "file", filePath)
if utils.IsArchive(filePath) {
basePath := filepath.Dir(filePath)
log.Info().Msgf("File %q is an archive, uncompressing to %s", filePath, basePath)
xlog.Info("File is an archive, uncompressing", "file", filePath, "basePath", basePath)
if err := utils.ExtractArchive(filePath, basePath); err != nil {
log.Debug().Msgf("Failed decompressing %q: %s", filePath, err.Error())
xlog.Debug("Failed decompressing", "file", filePath, "error", err)
return err
}
}

6
pkg/functions/functions.go
View File

@@ -3,7 +3,7 @@ package functions
import (
"encoding/json"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -59,11 +59,11 @@ func (f Functions) ToJSONStructure(name, args string) JSONFunctionStructure {
err := json.Unmarshal(dat, &prop)
if err != nil {
log.Error().Err(err).Msg("error unmarshalling dat")
xlog.Error("error unmarshalling dat", "error", err)
}
err = json.Unmarshal(dat2, &defsD)
if err != nil {
log.Error().Err(err).Msg("error unmarshalling dat2")
xlog.Error("error unmarshalling dat2", "error", err)
}
if js.Defs == nil {
js.Defs = defsD

22
pkg/functions/parse.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/mudler/LocalAI/pkg/functions/grammars"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// @Description GrammarConfig contains configuration for grammar parsing
@@ -150,22 +150,22 @@ func (g FunctionsConfig) GrammarOptions() []func(o *grammars.GrammarOption) {
}
func CleanupLLMResult(llmresult string, functionConfig FunctionsConfig) string {
log.Debug().Msgf("LLM result: %s", llmresult)
xlog.Debug("LLM result", "result", llmresult)
for _, item := range functionConfig.ReplaceLLMResult {
k, v := item.Key, item.Value
log.Debug().Msgf("Replacing %s with %s", k, v)
xlog.Debug("Replacing", "key", k, "value", v)
re := regexp.MustCompile(k)
llmresult = re.ReplaceAllString(llmresult, v)
}
log.Debug().Msgf("LLM result(processed): %s", llmresult)
xlog.Debug("LLM result(processed)", "result", llmresult)
return llmresult
}
func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string {
log.Debug().Msgf("ParseTextContent: %s", llmresult)
log.Debug().Msgf("CaptureLLMResult: %s", functionConfig.CaptureLLMResult)
xlog.Debug("ParseTextContent", "result", llmresult)
xlog.Debug("CaptureLLMResult", "config", functionConfig.CaptureLLMResult)
for _, r := range functionConfig.CaptureLLMResult {
// We use a regex to extract the JSON object from the response
@@ -223,15 +223,15 @@ func ParseJSON(s string) ([]map[string]any, error) {
func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults {
log.Debug().Msgf("LLM result: %s", llmresult)
xlog.Debug("LLM result", "result", llmresult)
for _, item := range functionConfig.ReplaceFunctionResults {
k, v := item.Key, item.Value
log.Debug().Msgf("Replacing %s with %s", k, v)
xlog.Debug("Replacing", "key", k, "value", v)
re := regexp.MustCompile(k)
llmresult = re.ReplaceAllString(llmresult, v)
}
log.Debug().Msgf("LLM result(function cleanup): %s", llmresult)
xlog.Debug("LLM result(function cleanup)", "result", llmresult)
functionNameKey := defaultFunctionNameKey
functionArgumentsKey := defaultFunctionArgumentsKey
@@ -256,10 +256,10 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
ss, err := ParseJSON(s)
//err := json.Unmarshal([]byte(s), &ss)
if err != nil {
log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects")
xlog.Debug("unable to unmarshal llm result in a single object or an array of JSON objects", "error", err, "escapedLLMResult", s)
}
log.Debug().Msgf("Function return: %s %+v", s, ss)
xlog.Debug("Function return", "result", s, "parsed", ss)
for _, s := range ss {
// The grammar defines the function name as "function", while OpenAI returns "name"

56
pkg/model/initializers.go
View File

@@ -10,7 +10,7 @@ import (
grpc "github.com/mudler/LocalAI/pkg/grpc"
"github.com/phayes/freeport"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -50,7 +50,7 @@ const (
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
return func(modelID, modelName, modelFile string) (*Model, error) {
log.Debug().Msgf("Loading Model %s with gRPC (file: %s) (backend: %s): %+v", modelID, modelFile, backend, *o)
xlog.Debug("Loading Model with gRPC", "modelID", modelID, "file", modelFile, "backend", backend, "options", *o)
var client *Model
@@ -67,17 +67,17 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
if os.Getenv(env) == "" {
err := os.Setenv(env, ml.ModelPath)
if err != nil {
log.Error().Err(err).Str("name", env).Str("modelPath", ml.ModelPath).Msg("unable to set environment variable to modelPath")
xlog.Error("unable to set environment variable to modelPath", "error", err, "name", env, "modelPath", ml.ModelPath)
}
}
}
// Check if the backend is provided as external
if uri, ok := ml.GetAllExternalBackends(o)[backend]; ok {
log.Debug().Msgf("Loading external backend: %s", uri)
xlog.Debug("Loading external backend", "uri", uri)
// check if uri is a file or a address
if fi, err := os.Stat(uri); err == nil {
log.Debug().Msgf("external backend is file: %+v", fi)
xlog.Debug("external backend is file", "file", fi)
serverAddress, err := getFreeAddress()
if err != nil {
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
@@ -85,43 +85,43 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
// Make sure the process is executable
process, err := ml.startProcess(uri, modelID, serverAddress)
if err != nil {
log.Error().Err(err).Str("path", uri).Msg("failed to launch ")
xlog.Error("failed to launch", "error", err, "path", uri)
return nil, err
}
log.Debug().Msgf("GRPC Service Started")
xlog.Debug("GRPC Service Started")
client = NewModel(modelID, serverAddress, process)
} else {
log.Debug().Msg("external backend is a uri")
xlog.Debug("external backend is a uri")
// address
client = NewModel(modelID, uri, nil)
}
} else {
log.Error().Msgf("Backend not found: %s", backend)
xlog.Error("Backend not found", "backend", backend)
return nil, fmt.Errorf("backend not found: %s", backend)
}
log.Debug().Msgf("Wait for the service to start up")
log.Debug().Msgf("Options: %+v", o.gRPCOptions)
xlog.Debug("Wait for the service to start up")
xlog.Debug("Options", "options", o.gRPCOptions)
// Wait for the service to start up
ready := false
for i := 0; i < o.grpcAttempts; i++ {
alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background())
if alive {
log.Debug().Msgf("GRPC Service Ready")
xlog.Debug("GRPC Service Ready")
ready = true
break
}
if err != nil && i == o.grpcAttempts-1 {
log.Error().Err(err).Msg("failed starting/connecting to the gRPC service")
xlog.Error("failed starting/connecting to the gRPC service", "error", err)
}
time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second)
}
if !ready {
log.Debug().Msgf("GRPC Service NOT ready")
xlog.Debug("GRPC Service NOT ready")
if process := client.Process(); process != nil {
process.Stop()
}
@@ -133,7 +133,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
options.ModelFile = modelFile
options.ModelPath = ml.ModelPath
log.Debug().Msgf("GRPC: Loading model with options: %+v", options)
xlog.Debug("GRPC: Loading model with options", "options", options)
res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options)
if err != nil {
@@ -156,16 +156,16 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
o := NewOptions(opts...)
log.Info().Str("modelID", o.modelID).Str("backend", o.backendString).Str("o.model", o.model).Msg("BackendLoader starting")
xlog.Info("BackendLoader starting", "modelID", o.modelID, "backend", o.backendString, "model", o.model)
backend := strings.ToLower(o.backendString)
if realBackend, exists := Aliases[backend]; exists {
typeAlias, exists := TypeAlias[backend]
if exists {
log.Debug().Msgf("'%s' is a type alias of '%s' (%s)", backend, realBackend, typeAlias)
xlog.Debug("alias is a type alias", "alias", backend, "realBackend", realBackend, "type", typeAlias)
o.gRPCOptions.Type = typeAlias
} else {
log.Debug().Msgf("'%s' is an alias of '%s'", backend, realBackend)
xlog.Debug("alias", "alias", backend, "realBackend", realBackend)
}
backend = realBackend
@@ -174,9 +174,9 @@ func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err e
model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o))
if err != nil {
if stopErr := ml.StopGRPC(only(o.modelID));stopErr != nil {
log.Error().Err(stopErr).Str("model", o.modelID).Msg("error stopping model")
xlog.Error("error stopping model", "error", stopErr, "model", o.modelID)
}
log.Error().Str("modelID", o.modelID).Err(err).Msgf("Failed to load model %s with backend %s", o.modelID, o.backendString)
xlog.Error("Failed to load model", "modelID", o.modelID, "error", err, "backend", o.backendString)
return nil, err
}
@@ -209,7 +209,7 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
// Return earlier if we have a model already loaded
// (avoid looping through all the backends)
if m := ml.CheckIsLoaded(o.modelID); m != nil {
log.Debug().Msgf("Model '%s' already loaded", o.modelID)
xlog.Debug("Model already loaded", "model", o.modelID)
// Update last used time for LRU tracking
ml.updateModelLastUsed(m)
return m.GRPC(o.parallelRequests, ml.wd), nil
@@ -239,30 +239,30 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
}
if len(autoLoadBackends) == 0 {
log.Error().Msg("No backends found")
xlog.Error("No backends found")
return nil, fmt.Errorf("no backends found")
}
log.Debug().Msgf("Loading from the following backends (in order): %+v", autoLoadBackends)
xlog.Debug("Loading from the following backends (in order)", "backends", autoLoadBackends)
log.Info().Msgf("Trying to load the model '%s' with the backend '%s'", o.modelID, autoLoadBackends)
xlog.Info("Trying to load the model", "modelID", o.modelID, "backends", autoLoadBackends)
for _, key := range autoLoadBackends {
log.Info().Msgf("[%s] Attempting to load", key)
xlog.Info("Attempting to load", "backend", key)
options := append(opts, []Option{
WithBackendString(key),
}...)
model, modelerr := ml.backendLoader(options...)
if modelerr == nil && model != nil {
log.Info().Msgf("[%s] Loads OK", key)
xlog.Info("Loads OK", "backend", key)
return model, nil
} else if modelerr != nil {
err = errors.Join(err, fmt.Errorf("[%s]: %w", key, modelerr))
log.Info().Msgf("[%s] Fails: %s", key, modelerr.Error())
xlog.Info("Fails", "backend", key, "error", modelerr.Error())
} else if model == nil {
err = errors.Join(err, fmt.Errorf("backend %s returned no usable model", key))
log.Info().Msgf("[%s] Fails: %s", key, "backend returned no usable model")
xlog.Info("Fails", "backend", key, "error", "backend returned no usable model")
}
}

20
pkg/model/loader.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// new idea: what if we declare a struct of these here, and use a loop to check?
@@ -173,7 +173,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
if loadingChan, isLoading := ml.loading[modelID]; isLoading {
ml.mu.Unlock()
// Wait for the other goroutine to finish loading
log.Debug().Str("modelID", modelID).Msg("Waiting for model to be loaded by another request")
xlog.Debug("Waiting for model to be loaded by another request", "modelID", modelID)
<-loadingChan
// Now check if the model is loaded
ml.mu.Lock()
@@ -201,7 +201,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
// Load the model (this can take a long time, no lock held)
modelFile := filepath.Join(ml.ModelPath, modelName)
log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
xlog.Debug("Loading model in memory from file", "file", modelFile)
model, err := loader(modelID, modelName, modelFile)
if err != nil {
@@ -239,28 +239,28 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model {
return nil
}
log.Debug().Msgf("Model already loaded in memory: %s", s)
xlog.Debug("Model already loaded in memory", "model", s)
client := m.GRPC(false, ml.wd)
log.Debug().Msgf("Checking model availability (%s)", s)
xlog.Debug("Checking model availability", "model", s)
cTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
alive, err := client.HealthCheck(cTimeout)
if !alive {
log.Warn().Msgf("GRPC Model not responding: %s", err.Error())
log.Warn().Msgf("Deleting the process in order to recreate it")
xlog.Warn("GRPC Model not responding", "error", err)
xlog.Warn("Deleting the process in order to recreate it")
process := m.Process()
if process == nil {
log.Error().Msgf("Process not found for '%s' and the model is not responding anymore !", s)
xlog.Error("Process not found and the model is not responding anymore", "model", s)
return m
}
if !process.IsAlive() {
log.Debug().Msgf("GRPC Process is not responding: %s", s)
xlog.Debug("GRPC Process is not responding", "model", s)
// stop and delete the process, this forces to re-load the model and re-create again the service
err := ml.deleteProcess(s)
if err != nil {
log.Error().Err(err).Str("process", s).Msg("error stopping process")
xlog.Error("error stopping process", "error", err, "process", s)
}
return nil
}

32
pkg/model/process.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/hpcloud/tail"
"github.com/mudler/LocalAI/pkg/signals"
process "github.com/mudler/go-processmanager"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true"
@@ -20,7 +20,7 @@ var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "
func (ml *ModelLoader) deleteProcess(s string) error {
model, ok := ml.models[s]
if !ok {
log.Debug().Msgf("Model %s not found", s)
xlog.Debug("Model not found", "model", s)
return fmt.Errorf("model %s not found", s)
}
@@ -28,7 +28,7 @@ func (ml *ModelLoader) deleteProcess(s string) error {
retries := 1
for model.GRPC(false, ml.wd).IsBusy() {
log.Debug().Msgf("%s busy. Waiting.", s)
xlog.Debug("Model busy. Waiting.", "model", s)
dur := time.Duration(retries*2) * time.Second
if dur > retryTimeout {
dur = retryTimeout
@@ -37,23 +37,23 @@ func (ml *ModelLoader) deleteProcess(s string) error {
retries++
if retries > 10 && forceBackendShutdown {
log.Warn().Msgf("Model %s is still busy after %d retries. Forcing shutdown.", s, retries)
xlog.Warn("Model is still busy after retries. Forcing shutdown.", "model", s, "retries", retries)
break
}
}
log.Debug().Msgf("Deleting process %s", s)
xlog.Debug("Deleting process", "model", s)
process := model.Process()
if process == nil {
log.Error().Msgf("No process for %s", s)
xlog.Error("No process", "model", s)
// Nothing to do as there is no process
return nil
}
err := process.Stop()
if err != nil {
log.Error().Err(err).Msgf("(deleteProcess) error while deleting process %s", s)
xlog.Error("(deleteProcess) error while deleting process", "error", err, "model", s)
}
return err
@@ -95,16 +95,16 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
// Check first if it has executable permissions
if fi, err := os.Stat(grpcProcess); err == nil {
if fi.Mode()&0111 == 0 {
log.Debug().Msgf("Process %s is not executable. Making it executable.", grpcProcess)
xlog.Debug("Process is not executable. Making it executable.", "process", grpcProcess)
if err := os.Chmod(grpcProcess, 0700); err != nil {
return nil, err
}
}
}
log.Debug().Msgf("Loading GRPC Process: %s", grpcProcess)
xlog.Debug("Loading GRPC Process", "process", grpcProcess)
log.Debug().Msgf("GRPC Service for %s will be running at: '%s'", id, serverAddress)
xlog.Debug("GRPC Service will be running", "id", id, "address", serverAddress)
workDir, err := filepath.Abs(filepath.Dir(grpcProcess))
if err != nil {
@@ -128,31 +128,31 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
return grpcControlProcess, err
}
log.Debug().Msgf("GRPC Service state dir: %s", grpcControlProcess.StateDir())
xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir())
signals.RegisterGracefulTerminationHandler(func() {
err := grpcControlProcess.Stop()
if err != nil {
log.Error().Err(err).Msg("error while shutting down grpc process")
xlog.Error("error while shutting down grpc process", "error", err)
}
})
go func() {
t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true})
if err != nil {
log.Debug().Msgf("Could not tail stderr")
xlog.Debug("Could not tail stderr")
}
for line := range t.Lines {
log.Debug().Msgf("GRPC(%s): stderr %s", strings.Join([]string{id, serverAddress}, "-"), line.Text)
xlog.Debug("GRPC stderr", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
}
}()
go func() {
t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true})
if err != nil {
log.Debug().Msgf("Could not tail stdout")
xlog.Debug("Could not tail stdout")
}
for line := range t.Lines {
log.Debug().Msgf("GRPC(%s): stdout %s", strings.Join([]string{id, serverAddress}, "-"), line.Text)
xlog.Debug("GRPC stdout", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
}
}()

66
pkg/model/watchdog.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/mudler/LocalAI/pkg/xsysinfo"
process "github.com/mudler/go-processmanager"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// WatchDog tracks all the requests from GRPC clients.
@@ -113,7 +113,7 @@ func (wd *WatchDog) GetMemoryReclaimerSettings() (enabled bool, threshold float6
func (wd *WatchDog) Shutdown() {
wd.Lock()
defer wd.Unlock()
log.Info().Msg("[WatchDog] Shutting down watchdog")
xlog.Info("[WatchDog] Shutting down watchdog")
wd.stop <- true
}
@@ -191,7 +191,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
return 0
}
log.Debug().Int("current", currentCount).Int("pendingLoads", pendingLoads).Int("limit", wd.lruLimit).Int("toEvict", modelsToEvict).Msg("[WatchDog] LRU enforcement triggered")
xlog.Debug("[WatchDog] LRU enforcement triggered", "current", currentCount, "pendingLoads", pendingLoads, "limit", wd.lruLimit, "toEvict", modelsToEvict)
// Build a list of models sorted by last used time (oldest first)
var models []modelUsageInfo
@@ -217,7 +217,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
var modelsToShutdown []string
for i := 0; i < modelsToEvict && i < len(models); i++ {
m := models[i]
log.Info().Str("model", m.model).Time("lastUsed", m.lastUsed).Msg("[WatchDog] LRU evicting model")
xlog.Info("[WatchDog] LRU evicting model", "model", m.model, "lastUsed", m.lastUsed)
modelsToShutdown = append(modelsToShutdown, m.model)
// Clean up the maps while we have the lock
wd.untrack(m.address)
@@ -227,21 +227,21 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
// Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[WatchDog] error shutting down model during LRU eviction")
xlog.Error("[WatchDog] error shutting down model during LRU eviction", "error", err, "model", model)
}
log.Debug().Str("model", model).Msg("[WatchDog] LRU eviction complete")
xlog.Debug("[WatchDog] LRU eviction complete", "model", model)
}
return len(modelsToShutdown)
}
func (wd *WatchDog) Run() {
log.Info().Msg("[WatchDog] starting watchdog")
xlog.Info("[WatchDog] starting watchdog")
for {
select {
case <-wd.stop:
log.Info().Msg("[WatchDog] Stopping watchdog")
xlog.Info("[WatchDog] Stopping watchdog")
return
case <-time.After(wd.watchdogInterval):
// Check if any monitoring is enabled
@@ -252,7 +252,7 @@ func (wd *WatchDog) Run() {
wd.Unlock()
if !busyCheck && !idleCheck && !memoryCheck {
log.Info().Msg("[WatchDog] No checks enabled, stopping watchdog")
xlog.Info("[WatchDog] No checks enabled, stopping watchdog")
return
}
if busyCheck {
@@ -270,19 +270,19 @@ func (wd *WatchDog) Run() {
func (wd *WatchDog) checkIdle() {
wd.Lock()
log.Debug().Msg("[WatchDog] Watchdog checks for idle connections")
xlog.Debug("[WatchDog] Watchdog checks for idle connections")
// Collect models to shutdown while holding the lock
var modelsToShutdown []string
for address, t := range wd.idleTime {
log.Debug().Msgf("[WatchDog] %s: idle connection", address)
xlog.Debug("[WatchDog] idle connection", "address", address)
if time.Since(t) > wd.idletimeout {
log.Warn().Msgf("[WatchDog] Address %s is idle for too long, killing it", address)
xlog.Warn("[WatchDog] Address is idle for too long, killing it", "address", address)
model, ok := wd.addressModelMap[address]
if ok {
modelsToShutdown = append(modelsToShutdown, model)
} else {
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address)
xlog.Warn("[WatchDog] Address unresolvable", "address", address)
}
wd.untrack(address)
}
@@ -292,28 +292,28 @@ func (wd *WatchDog) checkIdle() {
// Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
}
log.Debug().Msgf("[WatchDog] model shut down: %s", model)
xlog.Debug("[WatchDog] model shut down", "model", model)
}
}
func (wd *WatchDog) checkBusy() {
wd.Lock()
log.Debug().Msg("[WatchDog] Watchdog checks for busy connections")
xlog.Debug("[WatchDog] Watchdog checks for busy connections")
// Collect models to shutdown while holding the lock
var modelsToShutdown []string
for address, t := range wd.busyTime {
log.Debug().Msgf("[WatchDog] %s: active connection", address)
xlog.Debug("[WatchDog] active connection", "address", address)
if time.Since(t) > wd.timeout {
model, ok := wd.addressModelMap[address]
if ok {
log.Warn().Msgf("[WatchDog] Model %s is busy for too long, killing it", model)
xlog.Warn("[WatchDog] Model is busy for too long, killing it", "model", model)
modelsToShutdown = append(modelsToShutdown, model)
} else {
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address)
xlog.Warn("[WatchDog] Address unresolvable", "address", address)
}
wd.untrack(address)
}
@@ -323,9 +323,9 @@ func (wd *WatchDog) checkBusy() {
// Now shutdown models without holding the watchdog lock to prevent deadlock
for _, model := range modelsToShutdown {
if err := wd.pm.ShutdownModel(model); err != nil {
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
}
log.Debug().Msgf("[WatchDog] model shut down: %s", model)
xlog.Debug("[WatchDog] model shut down", "model", model)
}
}
@@ -344,7 +344,7 @@ func (wd *WatchDog) checkMemory() {
// Get current memory usage (GPU if available, otherwise RAM)
aggregate := xsysinfo.GetResourceAggregateInfo()
if aggregate.TotalMemory == 0 {
log.Debug().Msg("[WatchDog] No memory information available for memory reclaimer")
xlog.Debug("[WatchDog] No memory information available for memory reclaimer")
return
}
@@ -356,20 +356,11 @@ func (wd *WatchDog) checkMemory() {
memoryType = "RAM"
}
log.Debug().
Str("type", memoryType).
Float64("usage_percent", aggregate.UsagePercent).
Float64("threshold_percent", thresholdPercent).
Int("loaded_models", modelCount).
Msg("[WatchDog] Memory check")
xlog.Debug("[WatchDog] Memory check", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent, "loaded_models", modelCount)
// Check if usage exceeds threshold
if aggregate.UsagePercent > thresholdPercent {
log.Warn().
Str("type", memoryType).
Float64("usage_percent", aggregate.UsagePercent).
Float64("threshold_percent", thresholdPercent).
Msg("[WatchDog] Memory usage exceeds threshold, evicting LRU backend")
xlog.Warn("[WatchDog] Memory usage exceeds threshold, evicting LRU backend", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent)
// Evict the least recently used model
wd.evictLRUModel()
@@ -411,10 +402,7 @@ func (wd *WatchDog) evictLRUModel() {
// Get the LRU model
lruModel := models[0]
log.Info().
Str("model", lruModel.model).
Time("lastUsed", lruModel.lastUsed).
Msg("[WatchDog] Memory reclaimer evicting LRU model")
xlog.Info("[WatchDog] Memory reclaimer evicting LRU model", "model", lruModel.model, "lastUsed", lruModel.lastUsed)
// Untrack the model
wd.untrack(lruModel.address)
@@ -422,9 +410,9 @@ func (wd *WatchDog) evictLRUModel() {
// Shutdown the model
if err := wd.pm.ShutdownModel(lruModel.model); err != nil {
log.Error().Err(err).Str("model", lruModel.model).Msg("[WatchDog] error shutting down model during memory reclamation")
xlog.Error("[WatchDog] error shutting down model during memory reclamation", "error", err, "model", lruModel.model)
} else {
log.Info().Str("model", lruModel.model).Msg("[WatchDog] Memory reclaimer eviction complete")
xlog.Info("[WatchDog] Memory reclaimer eviction complete", "model", lruModel.model)
}
}

22
pkg/system/capabilities.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
"github.com/jaypipes/ghw/pkg/gpu"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
const (
@@ -49,11 +49,11 @@ func (s *SystemState) Capability(capMap map[string]string) string {
// Check if the reported capability is in the map
if _, exists := capMap[reportedCapability]; exists {
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("Using reported capability")
xlog.Debug("Using reported capability", "reportedCapability", reportedCapability, "capMap", capMap)
return reportedCapability
}
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("The requested capability was not found, using default capability")
xlog.Debug("The requested capability was not found, using default capability", "reportedCapability", reportedCapability, "capMap", capMap)
// Otherwise, return the default capability (catch-all)
return defaultCapability
}
@@ -61,7 +61,7 @@ func (s *SystemState) Capability(capMap map[string]string) string {
func (s *SystemState) getSystemCapabilities() string {
capability := os.Getenv(capabilityEnv)
if capability != "" {
log.Info().Str("capability", capability).Msgf("Using forced capability from environment variable (%s)", capabilityEnv)
xlog.Info("Using forced capability from environment variable", "capability", capability, "env", capabilityEnv)
return capability
}
@@ -77,27 +77,27 @@ func (s *SystemState) getSystemCapabilities() string {
if _, err := os.Stat(capabilityRunFile); err == nil {
capability, err := os.ReadFile(capabilityRunFile)
if err == nil {
log.Info().Str("capabilityRunFile", capabilityRunFile).Str("capability", string(capability)).Msgf("Using forced capability run file (%s)", capabilityRunFileEnv)
xlog.Info("Using forced capability run file", "capabilityRunFile", capabilityRunFile, "capability", string(capability), "env", capabilityRunFileEnv)
return strings.Trim(strings.TrimSpace(string(capability)), "\n")
}
}
// If we are on mac and arm64, we will return metal
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
log.Info().Msgf("Using metal capability (arm64 on mac), set %s to override", capabilityEnv)
xlog.Info("Using metal capability (arm64 on mac)", "env", capabilityEnv)
return metal
}
// If we are on mac and x86, we will return darwin-x86
if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
log.Info().Msgf("Using darwin-x86 capability (amd64 on mac), set %s to override", capabilityEnv)
xlog.Info("Using darwin-x86 capability (amd64 on mac)", "env", capabilityEnv)
return darwinX86
}
// If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t
if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" {
if s.GPUVendor == nvidia {
log.Info().Msgf("Using nvidia-l4t capability (arm64 on linux), set %s to override", capabilityEnv)
xlog.Info("Using nvidia-l4t capability (arm64 on linux)", "env", capabilityEnv)
if cuda13DirExists {
return nvidiaL4TCuda13
}
@@ -117,14 +117,14 @@ func (s *SystemState) getSystemCapabilities() string {
}
if s.GPUVendor == "" {
log.Info().Msgf("Default capability (no GPU detected), set %s to override", capabilityEnv)
xlog.Info("Default capability (no GPU detected)", "env", capabilityEnv)
return defaultCapability
}
log.Info().Str("Capability", s.GPUVendor).Msgf("Capability automatically detected, set %s to override", capabilityEnv)
xlog.Info("Capability automatically detected", "capability", s.GPUVendor, "env", capabilityEnv)
// If vram is less than 4GB, let's default to CPU but warn the user that they can override that via env
if s.VRAM <= 4*1024*1024*1024 {
log.Warn().Msgf("VRAM is less than 4GB, defaulting to CPU. Set %s to override", capabilityEnv)
xlog.Warn("VRAM is less than 4GB, defaulting to CPU", "env", capabilityEnv)
return defaultCapability
}

8
pkg/system/state.go
View File

@@ -3,7 +3,7 @@ package system
import (
"github.com/jaypipes/ghw/pkg/gpu"
"github.com/mudler/LocalAI/pkg/xsysinfo"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
type Backend struct {
@@ -51,11 +51,11 @@ func GetSystemState(opts ...SystemStateOptions) (*SystemState, error) {
// Detection is best-effort here, we don't want to fail if it fails
state.gpus, _ = xsysinfo.GPUs()
log.Debug().Any("gpus", state.gpus).Msg("GPUs")
xlog.Debug("GPUs", "gpus", state.gpus)
state.GPUVendor, _ = detectGPUVendor(state.gpus)
log.Debug().Str("gpuVendor", state.GPUVendor).Msg("GPU vendor")
xlog.Debug("GPU vendor", "gpuVendor", state.GPUVendor)
state.VRAM, _ = xsysinfo.TotalAvailableVRAM()
log.Debug().Any("vram", state.VRAM).Msg("Total available VRAM")
xlog.Debug("Total available VRAM", "vram", state.VRAM)
return state, nil
}

102
pkg/utils/base64.go
View File

@@ -1,51 +1,51 @@
package utils
import (
"encoding/base64"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"github.com/rs/zerolog/log"
)
var base64DownloadClient http.Client = http.Client{
Timeout: 30 * time.Second,
}
var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
// GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers
func GetContentURIAsBase64(s string) (string, error) {
if strings.HasPrefix(s, "http") || strings.HasPrefix(s, "https") {
// download the image
resp, err := base64DownloadClient.Get(s)
if err != nil {
return "", err
}
defer resp.Body.Close()
// read the image data into memory
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
// encode the image data in base64
encoded := base64.StdEncoding.EncodeToString(data)
// return the base64 string
return encoded, nil
}
// Match any data URI prefix pattern
if match := dataURIPattern.FindString(s); match != "" {
log.Debug().Msgf("Found data URI prefix: %s", match)
return strings.Replace(s, match, "", 1), nil
}
return "", fmt.Errorf("not valid base64 data type string")
}
package utils
import (
"encoding/base64"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"github.com/mudler/xlog"
)
var base64DownloadClient http.Client = http.Client{
Timeout: 30 * time.Second,
}
var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
// GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers
func GetContentURIAsBase64(s string) (string, error) {
if strings.HasPrefix(s, "http") || strings.HasPrefix(s, "https") {
// download the image
resp, err := base64DownloadClient.Get(s)
if err != nil {
return "", err
}
defer resp.Body.Close()
// read the image data into memory
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
// encode the image data in base64
encoded := base64.StdEncoding.EncodeToString(data)
// return the base64 string
return encoded, nil
}
// Match any data URI prefix pattern
if match := dataURIPattern.FindString(s); match != "" {
xlog.Debug("Found data URI prefix", "prefix", match)
return strings.Replace(s, match, "", 1), nil
}
return "", fmt.Errorf("not valid base64 data type string")
}

6
pkg/utils/logging.go
View File

@@ -3,7 +3,7 @@ package utils
import (
"time"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
var lastProgress time.Time = time.Now()
@@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
}
if total != "" {
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
xlog.Info("Downloading", "fileName", fileName, "current", current, "total", total, "percentage", percentage, "eta", eta)
} else {
log.Info().Msgf("Downloading: %s", current)
xlog.Info("Downloading", "current", current)
}
}
}

29
pkg/xsysinfo/gpu.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/jaypipes/ghw"
"github.com/jaypipes/ghw/pkg/gpu"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// GPU vendor constants
@@ -213,7 +213,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("nvidia-smi failed")
xlog.Debug("nvidia-smi failed", "error", err, "stderr", stderr.String())
return nil
}
@@ -246,7 +246,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
// Unified memory device - fall back to system RAM
sysInfo, err := GetSystemRAMInfo()
if err != nil {
log.Debug().Err(err).Str("device", name).Msg("failed to get system RAM for unified memory device")
xlog.Debug("failed to get system RAM for unified memory device", "error", err, "device", name)
// Still add the GPU but with zero memory info
gpus = append(gpus, GPUMemoryInfo{
Index: idx,
@@ -267,13 +267,10 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
usagePercent = float64(usedBytes) / float64(totalBytes) * 100
}
log.Debug().
Str("device", name).
Uint64("system_ram_bytes", totalBytes).
Msg("using system RAM for unified memory GPU")
xlog.Debug("using system RAM for unified memory GPU", "device", name, "system_ram_bytes", totalBytes)
} else if isNA {
// Unknown device with N/A values - skip memory info
log.Debug().Str("device", name).Msg("nvidia-smi returned N/A for unknown device")
xlog.Debug("nvidia-smi returned N/A for unknown device", "device", name)
gpus = append(gpus, GPUMemoryInfo{
Index: idx,
Name: name,
@@ -329,7 +326,7 @@ func getAMDGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("rocm-smi failed")
xlog.Debug("rocm-smi failed", "error", err, "stderr", stderr.String())
return nil
}
@@ -416,7 +413,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("xpu-smi discovery failed")
xlog.Debug("xpu-smi discovery failed", "error", err, "stderr", stderr.String())
return nil
}
@@ -431,7 +428,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
}
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse xpu-smi discovery output")
xlog.Debug("failed to parse xpu-smi discovery output", "error", err)
return nil
}
@@ -494,7 +491,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("intel_gpu_top failed")
xlog.Debug("intel_gpu_top failed", "error", err, "stderr", stderr.String())
return nil
}
@@ -523,7 +520,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
}
if err := json.Unmarshal([]byte(lastJSON), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse intel_gpu_top output")
xlog.Debug("failed to parse intel_gpu_top output", "error", err)
return nil
}
@@ -557,7 +554,7 @@ func GetResourceInfo() ResourceInfo {
// No GPU - fall back to system RAM
ramInfo, err := GetSystemRAMInfo()
if err != nil {
log.Debug().Err(err).Msg("failed to get system RAM info")
xlog.Debug("failed to get system RAM info", "error", err)
return ResourceInfo{
Type: "ram",
Available: false,
@@ -601,7 +598,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("vulkaninfo failed")
xlog.Debug("vulkaninfo failed", "error", err, "stderr", stderr.String())
return nil
}
@@ -620,7 +617,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
}
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
log.Debug().Err(err).Msg("failed to parse vulkaninfo output")
xlog.Debug("failed to parse vulkaninfo output", "error", err)
return nil
}

4
pkg/xsysinfo/memory.go
View File

@@ -2,7 +2,7 @@ package xsysinfo
import (
"github.com/mudler/memory"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
// SystemRAMInfo contains system RAM usage information
@@ -25,7 +25,7 @@ func GetSystemRAMInfo() (*SystemRAMInfo, error) {
if total > 0 {
usagePercent = float64(used) / float64(total) * 100
}
log.Debug().Uint64("total", total).Uint64("used", used).Uint64("free", free).Float64("usage_percent", usagePercent).Msg("System RAM Info")
xlog.Debug("System RAM Info", "total", total, "used", used, "free", free, "usage_percent", usagePercent)
return &SystemRAMInfo{
Total: total,
Used: used,

5
tests/integration/integration_suite_test.go
View File

@@ -6,12 +6,11 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
)
func TestLocalAI(t *testing.T) {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
RegisterFailHandler(Fail)
RunSpecs(t, "LocalAI test suite")
}

7
tests/integration/stores_test.go
View File

@@ -8,8 +8,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/mudler/xlog"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/grpc"
@@ -187,7 +186,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
for i, k := range keys {
s := sims[i]
log.Debug().Float32("similarity", s).Msgf("key: %v", k)
xlog.Debug("key", "similarity", s, "key", k)
}
Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5}))
@@ -214,7 +213,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
for i, k := range ks {
s := sims[i]
log.Debug().Float32("similarity", s).Msgf("key: %v", k)
xlog.Debug("key", "similarity", s, "key", k)
}
Expect(ks[0]).To(Equal(keys[0]))