mirror of
https://github.com/mudler/LocalAI.git
synced 2026-04-29 11:37:40 -04:00
chore(refactor): move logging to common package based on slog (#7668)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
committed by
GitHub
parent
38cde81ff4
commit
c37785b78c
@@ -4,11 +4,11 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func assert(cond bool, msg string) {
|
func assert(cond bool, msg string) {
|
||||||
if !cond {
|
if !cond {
|
||||||
log.Fatal().Stack().Msg(msg)
|
xlog.Fatal().Stack().Msg(msg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,11 +4,9 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
"os"
|
|
||||||
|
|
||||||
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||||
"github.com/rs/zerolog"
|
"github.com/mudler/xlog"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -16,7 +14,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/grpc/base"
|
"github.com/mudler/LocalAI/pkg/grpc/base"
|
||||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Store struct {
|
type Store struct {
|
||||||
@@ -135,7 +135,7 @@ func (s *Store) StoresSet(opts *pb.StoresSetOptions) error {
|
|||||||
} else {
|
} else {
|
||||||
sample = k.Floats
|
sample = k.Floats
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Key is not normalized: %v", sample)
|
xlog.Debug("Key is not normalized", "sample", sample)
|
||||||
}
|
}
|
||||||
|
|
||||||
kvs[i] = Pair{
|
kvs[i] = Pair{
|
||||||
@@ -238,7 +238,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
|
|||||||
assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k))
|
assert(!hasKey(s.keys, k), fmt.Sprintf("Key exists, but was not found: t=%d, %v", len(tail_ks), k))
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Delete: found = %v, t = %d, j = %d, len(merge_ks) = %d, len(merge_vs) = %d", found, len(tail_ks), j, len(merge_ks), len(merge_vs))
|
xlog.Debug("Delete", "found", found, "tailLen", len(tail_ks), "j", j, "mergeKeysLen", len(merge_ks), "mergeValuesLen", len(merge_vs))
|
||||||
}
|
}
|
||||||
|
|
||||||
merge_ks = append(merge_ks, tail_ks...)
|
merge_ks = append(merge_ks, tail_ks...)
|
||||||
@@ -261,7 +261,7 @@ func (s *Store) StoresDelete(opts *pb.StoresDeleteOptions) error {
|
|||||||
}(), "Keys to delete still present")
|
}(), "Keys to delete still present")
|
||||||
|
|
||||||
if len(s.keys) != l {
|
if len(s.keys) != l {
|
||||||
log.Debug().Msgf("Delete: Some keys not found: len(s.keys) = %d, l = %d", len(s.keys), l)
|
xlog.Debug("Delete: Some keys not found", "keysLen", len(s.keys), "expectedLen", l)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -273,7 +273,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
|
|||||||
ks := sortIntoKeySlicese(opts.Keys)
|
ks := sortIntoKeySlicese(opts.Keys)
|
||||||
|
|
||||||
if len(s.keys) == 0 {
|
if len(s.keys) == 0 {
|
||||||
log.Debug().Msgf("Get: No keys in store")
|
xlog.Debug("Get: No keys in store")
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.keyLen == -1 {
|
if s.keyLen == -1 {
|
||||||
@@ -305,7 +305,7 @@ func (s *Store) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(pbKeys) != len(opts.Keys) {
|
if len(pbKeys) != len(opts.Keys) {
|
||||||
log.Debug().Msgf("Get: Some keys not found: len(pbKeys) = %d, len(opts.Keys) = %d, len(s.Keys) = %d", len(pbKeys), len(opts.Keys), len(s.keys))
|
xlog.Debug("Get: Some keys not found", "pbKeysLen", len(pbKeys), "optsKeysLen", len(opts.Keys), "storeKeysLen", len(s.keys))
|
||||||
}
|
}
|
||||||
|
|
||||||
return pb.StoresGetResult{
|
return pb.StoresGetResult{
|
||||||
@@ -507,7 +507,7 @@ func (s *Store) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, err
|
|||||||
} else {
|
} else {
|
||||||
sample = tk
|
sample = tk
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Trying to compare non-normalized key with normalized keys: %v", sample)
|
xlog.Debug("Trying to compare non-normalized key with normalized keys", "sample", sample)
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.StoresFindFallback(opts)
|
return s.StoresFindFallback(opts)
|
||||||
|
|||||||
@@ -8,10 +8,7 @@ import (
|
|||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
"github.com/mudler/LocalAI/core/cli"
|
"github.com/mudler/LocalAI/core/cli"
|
||||||
"github.com/mudler/LocalAI/internal"
|
"github.com/mudler/LocalAI/internal"
|
||||||
"github.com/mudler/cogito/pkg/xlog"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
|
|
||||||
_ "github.com/mudler/LocalAI/swagger"
|
_ "github.com/mudler/LocalAI/swagger"
|
||||||
)
|
)
|
||||||
@@ -19,9 +16,8 @@ import (
|
|||||||
func main() {
|
func main() {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Initialize zerolog at a level of INFO, we will set the desired level after we parse the CLI options
|
// Initialize xlog at a level of INFO, we will set the desired level after we parse the CLI options
|
||||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
|
||||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
|
||||||
|
|
||||||
// handle loading environment variables from .env files
|
// handle loading environment variables from .env files
|
||||||
envFiles := []string{".env", "localai.env"}
|
envFiles := []string{".env", "localai.env"}
|
||||||
@@ -33,10 +29,10 @@ func main() {
|
|||||||
|
|
||||||
for _, envFile := range envFiles {
|
for _, envFile := range envFiles {
|
||||||
if _, err := os.Stat(envFile); err == nil {
|
if _, err := os.Stat(envFile); err == nil {
|
||||||
log.Debug().Str("envFile", envFile).Msg("env file found, loading environment variables from file")
|
xlog.Debug("env file found, loading environment variables from file", "envFile", envFile)
|
||||||
err = godotenv.Load(envFile)
|
err = godotenv.Load(envFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("envFile", envFile).Msg("failed to load environment variables from file")
|
xlog.Error("failed to load environment variables from file", "error", err, "envFile", envFile)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -68,7 +64,6 @@ Version: ${version}
|
|||||||
logLevel := "info"
|
logLevel := "info"
|
||||||
if cli.CLI.Debug && cli.CLI.LogLevel == nil {
|
if cli.CLI.Debug && cli.CLI.LogLevel == nil {
|
||||||
logLevel = "debug"
|
logLevel = "debug"
|
||||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
|
||||||
cli.CLI.LogLevel = &logLevel
|
cli.CLI.LogLevel = &logLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,31 +71,12 @@ Version: ${version}
|
|||||||
cli.CLI.LogLevel = &logLevel
|
cli.CLI.LogLevel = &logLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set cogito logger to the same level as our logger
|
// Set xlog logger with the desired level and text format
|
||||||
// Leave an empty format type
|
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), "text"))
|
||||||
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), ""))
|
|
||||||
|
|
||||||
switch *cli.CLI.LogLevel {
|
|
||||||
case "error":
|
|
||||||
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
|
|
||||||
log.Debug().Msg("Setting logging to error")
|
|
||||||
case "warn":
|
|
||||||
zerolog.SetGlobalLevel(zerolog.WarnLevel)
|
|
||||||
log.Debug().Msg("Setting logging to warn")
|
|
||||||
case "info":
|
|
||||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
|
||||||
log.Debug().Msg("Setting logging to info")
|
|
||||||
case "debug":
|
|
||||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
|
||||||
log.Debug().Msg("Setting logging to debug")
|
|
||||||
case "trace":
|
|
||||||
zerolog.SetGlobalLevel(zerolog.TraceLevel)
|
|
||||||
log.Debug().Msg("Setting logging to trace")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the thing!
|
// Run the thing!
|
||||||
err = ctx.Run(&cli.CLI.Context)
|
err = ctx.Run(&cli.CLI.Context)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Err(err).Msg("Error running the application")
|
xlog.Fatal("Error running the application", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RestartAgentJobService restarts the agent job service with current ApplicationConfig settings
|
// RestartAgentJobService restarts the agent job service with current ApplicationConfig settings
|
||||||
@@ -15,7 +15,7 @@ func (a *Application) RestartAgentJobService() error {
|
|||||||
// Stop existing service if running
|
// Stop existing service if running
|
||||||
if a.agentJobService != nil {
|
if a.agentJobService != nil {
|
||||||
if err := a.agentJobService.Stop(); err != nil {
|
if err := a.agentJobService.Stop(); err != nil {
|
||||||
log.Warn().Err(err).Msg("Error stopping agent job service")
|
xlog.Warn("Error stopping agent job service", "error", err)
|
||||||
}
|
}
|
||||||
// Wait a bit for shutdown to complete
|
// Wait a bit for shutdown to complete
|
||||||
time.Sleep(200 * time.Millisecond)
|
time.Sleep(200 * time.Millisecond)
|
||||||
@@ -32,12 +32,11 @@ func (a *Application) RestartAgentJobService() error {
|
|||||||
// Start the service
|
// Start the service
|
||||||
err := agentJobService.Start(a.ApplicationConfig().Context)
|
err := agentJobService.Start(a.ApplicationConfig().Context)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to start agent job service")
|
xlog.Error("Failed to start agent job service", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
a.agentJobService = agentJobService
|
a.agentJobService = agentJobService
|
||||||
log.Info().Msg("Agent job service restarted")
|
xlog.Info("Agent job service restarted")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"dario.cat/mergo"
|
"dario.cat/mergo"
|
||||||
"github.com/fsnotify/fsnotify"
|
"github.com/fsnotify/fsnotify"
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
|
type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
|
||||||
@@ -33,15 +33,15 @@ func newConfigFileHandler(appConfig *config.ApplicationConfig) configFileHandler
|
|||||||
}
|
}
|
||||||
err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
|
err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("file", "api_keys.json").Msg("unable to register config file handler")
|
xlog.Error("unable to register config file handler", "error", err, "file", "api_keys.json")
|
||||||
}
|
}
|
||||||
err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
|
err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("file", "external_backends.json").Msg("unable to register config file handler")
|
xlog.Error("unable to register config file handler", "error", err, "file", "external_backends.json")
|
||||||
}
|
}
|
||||||
err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true)
|
err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("file", "runtime_settings.json").Msg("unable to register config file handler")
|
xlog.Error("unable to register config file handler", "error", err, "file", "runtime_settings.json")
|
||||||
}
|
}
|
||||||
// Note: agent_tasks.json and agent_jobs.json are handled by AgentJobService directly
|
// Note: agent_tasks.json and agent_jobs.json are handled by AgentJobService directly
|
||||||
// The service watches and reloads these files internally
|
// The service watches and reloads these files internally
|
||||||
@@ -62,14 +62,14 @@ func (c *configFileHandler) Register(filename string, handler fileHandler, runNo
|
|||||||
|
|
||||||
func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
|
func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
|
||||||
rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
|
rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
|
||||||
log.Trace().Str("filename", rootedFilePath).Msg("reading file for dynamic config update")
|
xlog.Debug("reading file for dynamic config update", "filename", rootedFilePath)
|
||||||
fileContent, err := os.ReadFile(rootedFilePath)
|
fileContent, err := os.ReadFile(rootedFilePath)
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
log.Error().Err(err).Str("filename", rootedFilePath).Msg("could not read file")
|
xlog.Error("could not read file", "error", err, "filename", rootedFilePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = handler(fileContent, c.appConfig); err != nil {
|
if err = handler(fileContent, c.appConfig); err != nil {
|
||||||
log.Error().Err(err).Msg("WatchConfigDirectory goroutine failed to update options")
|
xlog.Error("WatchConfigDirectory goroutine failed to update options", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,13 +81,13 @@ func (c *configFileHandler) Watch() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if c.appConfig.DynamicConfigsDirPollInterval > 0 {
|
if c.appConfig.DynamicConfigsDirPollInterval > 0 {
|
||||||
log.Debug().Msg("Poll interval set, falling back to polling for configuration changes")
|
xlog.Debug("Poll interval set, falling back to polling for configuration changes")
|
||||||
ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
|
ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
<-ticker.C
|
<-ticker.C
|
||||||
for file, handler := range c.handlers {
|
for file, handler := range c.handlers {
|
||||||
log.Debug().Str("file", file).Msg("polling config file")
|
xlog.Debug("polling config file", "file", file)
|
||||||
c.callHandler(file, handler)
|
c.callHandler(file, handler)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,7 +111,7 @@ func (c *configFileHandler) Watch() error {
|
|||||||
c.callHandler(filepath.Base(event.Name), handler)
|
c.callHandler(filepath.Base(event.Name), handler)
|
||||||
}
|
}
|
||||||
case err, ok := <-c.watcher.Errors:
|
case err, ok := <-c.watcher.Errors:
|
||||||
log.Error().Err(err).Msg("config watcher error received")
|
xlog.Error("config watcher error received", "error", err)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -135,8 +135,7 @@ func (c *configFileHandler) Stop() error {
|
|||||||
|
|
||||||
func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
||||||
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
||||||
log.Debug().Msg("processing api keys runtime update")
|
xlog.Debug("processing api keys runtime update", "numKeys", len(startupAppConfig.ApiKeys))
|
||||||
log.Trace().Int("numKeys", len(startupAppConfig.ApiKeys)).Msg("api keys provided at startup")
|
|
||||||
|
|
||||||
if len(fileContent) > 0 {
|
if len(fileContent) > 0 {
|
||||||
// Parse JSON content from the file
|
// Parse JSON content from the file
|
||||||
@@ -146,14 +145,14 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Trace().Int("numKeys", len(fileKeys)).Msg("discovered API keys from api keys dynamic config dile")
|
xlog.Debug("discovered API keys from api keys dynamic config file", "numKeys", len(fileKeys))
|
||||||
|
|
||||||
appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
|
appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
|
||||||
} else {
|
} else {
|
||||||
log.Trace().Msg("no API keys discovered from dynamic config file")
|
xlog.Debug("no API keys discovered from dynamic config file")
|
||||||
appConfig.ApiKeys = startupAppConfig.ApiKeys
|
appConfig.ApiKeys = startupAppConfig.ApiKeys
|
||||||
}
|
}
|
||||||
log.Trace().Int("numKeys", len(appConfig.ApiKeys)).Msg("total api keys after processing")
|
xlog.Debug("total api keys after processing", "numKeys", len(appConfig.ApiKeys))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,7 +161,7 @@ func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
|||||||
|
|
||||||
func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
||||||
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
||||||
log.Debug().Msg("processing external_backends.json")
|
xlog.Debug("processing external_backends.json")
|
||||||
|
|
||||||
if len(fileContent) > 0 {
|
if len(fileContent) > 0 {
|
||||||
// Parse JSON content from the file
|
// Parse JSON content from the file
|
||||||
@@ -179,7 +178,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
|
|||||||
} else {
|
} else {
|
||||||
appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
|
appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
|
||||||
}
|
}
|
||||||
log.Debug().Msg("external backends loaded from external_backends.json")
|
xlog.Debug("external backends loaded from external_backends.json")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return handler
|
return handler
|
||||||
@@ -187,7 +186,7 @@ func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHan
|
|||||||
|
|
||||||
func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler {
|
||||||
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
|
||||||
log.Debug().Msg("processing runtime_settings.json")
|
xlog.Debug("processing runtime_settings.json")
|
||||||
|
|
||||||
// Determine if settings came from env vars by comparing with startup config
|
// Determine if settings came from env vars by comparing with startup config
|
||||||
// startupAppConfig contains the original values set from env vars at startup.
|
// startupAppConfig contains the original values set from env vars at startup.
|
||||||
@@ -241,7 +240,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
appConfig.WatchDogIdleTimeout = dur
|
appConfig.WatchDogIdleTimeout = dur
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
|
xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout {
|
if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout {
|
||||||
@@ -249,7 +248,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
appConfig.WatchDogBusyTimeout = dur
|
appConfig.WatchDogBusyTimeout = dur
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
|
xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Handle MaxActiveBackends (new) and SingleBackend (deprecated)
|
// Handle MaxActiveBackends (new) and SingleBackend (deprecated)
|
||||||
@@ -340,7 +339,7 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Debug().Msg("runtime settings loaded from runtime_settings.json")
|
xlog.Debug("runtime settings loaded from runtime_settings.json")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return handler
|
return handler
|
||||||
|
|||||||
@@ -14,8 +14,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
|
|
||||||
"github.com/mudler/edgevpn/pkg/node"
|
"github.com/mudler/edgevpn/pkg/node"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
zlog "github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *Application) StopP2P() error {
|
func (a *Application) StopP2P() error {
|
||||||
@@ -86,14 +85,14 @@ func (a *Application) StartP2P() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Attach a ServiceDiscoverer to the p2p node
|
// Attach a ServiceDiscoverer to the p2p node
|
||||||
log.Info().Msg("Starting P2P server discovery...")
|
xlog.Info("Starting P2P server discovery...")
|
||||||
if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
|
if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
|
||||||
var tunnelAddresses []string
|
var tunnelAddresses []string
|
||||||
for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
|
for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
|
||||||
if v.IsOnline() {
|
if v.IsOnline() {
|
||||||
tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
|
tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
|
||||||
} else {
|
} else {
|
||||||
log.Info().Msgf("Node %s is offline", v.ID)
|
xlog.Info("Node is offline", "node", v.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if a.applicationConfig.TunnelCallback != nil {
|
if a.applicationConfig.TunnelCallback != nil {
|
||||||
@@ -143,17 +142,17 @@ func (a *Application) RestartP2P() error {
|
|||||||
// Start P2P stack in a goroutine
|
// Start P2P stack in a goroutine
|
||||||
go func() {
|
go func() {
|
||||||
if err := a.StartP2P(); err != nil {
|
if err := a.StartP2P(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to start P2P stack")
|
xlog.Error("Failed to start P2P stack", "error", err)
|
||||||
cancel() // Cancel context on error
|
cancel() // Cancel context on error
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
log.Info().Msg("P2P stack restarted with new settings")
|
xlog.Info("P2P stack restarted with new settings")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func syncState(ctx context.Context, n *node.Node, app *Application) error {
|
func syncState(ctx context.Context, n *node.Node, app *Application) error {
|
||||||
zlog.Debug().Msg("[p2p-sync] Syncing state")
|
xlog.Debug("[p2p-sync] Syncing state")
|
||||||
|
|
||||||
whatWeHave := []string{}
|
whatWeHave := []string{}
|
||||||
for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
|
for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
|
||||||
@@ -162,20 +161,20 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
|
|||||||
|
|
||||||
ledger, _ := n.Ledger()
|
ledger, _ := n.Ledger()
|
||||||
currentData := ledger.CurrentData()
|
currentData := ledger.CurrentData()
|
||||||
zlog.Debug().Msgf("[p2p-sync] Current data: %v", currentData)
|
xlog.Debug("[p2p-sync] Current data", "data", currentData)
|
||||||
data, exists := ledger.GetKey("shared_state", "models")
|
data, exists := ledger.GetKey("shared_state", "models")
|
||||||
if !exists {
|
if !exists {
|
||||||
ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
|
ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
|
||||||
zlog.Debug().Msgf("No models found in the ledger, announced our models: %v", whatWeHave)
|
xlog.Debug("No models found in the ledger, announced our models", "models", whatWeHave)
|
||||||
}
|
}
|
||||||
|
|
||||||
models := []string{}
|
models := []string{}
|
||||||
if err := data.Unmarshal(&models); err != nil {
|
if err := data.Unmarshal(&models); err != nil {
|
||||||
zlog.Warn().Err(err).Msg("error unmarshalling models")
|
xlog.Warn("error unmarshalling models", "error", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
zlog.Debug().Msgf("[p2p-sync] Models that are present in this instance: %v\nModels that are in the ledger: %v", whatWeHave, models)
|
xlog.Debug("[p2p-sync] Models comparison", "ourModels", whatWeHave, "ledgerModels", models)
|
||||||
|
|
||||||
// Sync with our state
|
// Sync with our state
|
||||||
whatIsNotThere := []string{}
|
whatIsNotThere := []string{}
|
||||||
@@ -185,7 +184,7 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(whatIsNotThere) > 0 {
|
if len(whatIsNotThere) > 0 {
|
||||||
zlog.Debug().Msgf("[p2p-sync] Announcing our models: %v", append(models, whatIsNotThere...))
|
xlog.Debug("[p2p-sync] Announcing our models", "models", append(models, whatIsNotThere...))
|
||||||
ledger.AnnounceUpdate(
|
ledger.AnnounceUpdate(
|
||||||
ctx,
|
ctx,
|
||||||
1*time.Minute,
|
1*time.Minute,
|
||||||
@@ -198,16 +197,16 @@ func syncState(ctx context.Context, n *node.Node, app *Application) error {
|
|||||||
// Check if we have a model that is not in our state, otherwise install it
|
// Check if we have a model that is not in our state, otherwise install it
|
||||||
for _, model := range models {
|
for _, model := range models {
|
||||||
if slices.Contains(whatWeHave, model) {
|
if slices.Contains(whatWeHave, model) {
|
||||||
zlog.Debug().Msgf("[p2p-sync] Model %s is already present in this instance", model)
|
xlog.Debug("[p2p-sync] Model is already present in this instance", "model", model)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// we install model
|
// we install model
|
||||||
zlog.Info().Msgf("[p2p-sync] Installing model which is not present in this instance: %s", model)
|
xlog.Info("[p2p-sync] Installing model which is not present in this instance", "model", model)
|
||||||
|
|
||||||
uuid, err := uuid.NewUUID()
|
uuid, err := uuid.NewUUID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zlog.Error().Err(err).Msg("error generating UUID")
|
xlog.Error("error generating UUID", "error", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,7 +229,7 @@ func (a *Application) p2pSync(ctx context.Context, n *node.Node) error {
|
|||||||
return
|
return
|
||||||
case <-time.After(1 * time.Minute):
|
case <-time.After(1 * time.Minute):
|
||||||
if err := syncState(ctx, n, a); err != nil {
|
if err := syncState(ctx, n, a); err != nil {
|
||||||
zlog.Error().Err(err).Msg("error syncing state")
|
xlog.Error("error syncing state", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func New(opts ...config.AppOption) (*Application, error) {
|
func New(opts ...config.AppOption) (*Application, error) {
|
||||||
@@ -28,8 +28,8 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
application := newApplication(options)
|
application := newApplication(options)
|
||||||
application.startupConfig = &startupConfigCopy
|
application.startupConfig = &startupConfigCopy
|
||||||
|
|
||||||
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.SystemState.Model.ModelsPath)
|
xlog.Info("Starting LocalAI", "threads", options.Threads, "modelsPath", options.SystemState.Model.ModelsPath)
|
||||||
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
|
xlog.Info("LocalAI version", "version", internal.PrintableVersion())
|
||||||
|
|
||||||
if err := application.start(); err != nil {
|
if err := application.start(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -37,14 +37,14 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
|
|
||||||
caps, err := xsysinfo.CPUCapabilities()
|
caps, err := xsysinfo.CPUCapabilities()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Debug().Msgf("CPU capabilities: %v", caps)
|
xlog.Debug("CPU capabilities", "capabilities", caps)
|
||||||
|
|
||||||
}
|
}
|
||||||
gpus, err := xsysinfo.GPUs()
|
gpus, err := xsysinfo.GPUs()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Debug().Msgf("GPU count: %d", len(gpus))
|
xlog.Debug("GPU count", "count", len(gpus))
|
||||||
for _, gpu := range gpus {
|
for _, gpu := range gpus {
|
||||||
log.Debug().Msgf("GPU: %s", gpu.String())
|
xlog.Debug("GPU", "gpu", gpu.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,33 +71,33 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := coreStartup.InstallModels(options.Context, application.GalleryService(), options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
|
if err := coreStartup.InstallModels(options.Context, application.GalleryService(), options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
|
||||||
log.Error().Err(err).Msg("error installing models")
|
xlog.Error("error installing models", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, backend := range options.ExternalBackends {
|
for _, backend := range options.ExternalBackends {
|
||||||
if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
|
if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
|
||||||
log.Error().Err(err).Msg("error installing external backend")
|
xlog.Error("error installing external backend", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
configLoaderOpts := options.ToConfigLoaderOptions()
|
configLoaderOpts := options.ToConfigLoaderOptions()
|
||||||
|
|
||||||
if err := application.ModelConfigLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil {
|
if err := application.ModelConfigLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil {
|
||||||
log.Error().Err(err).Msg("error loading config files")
|
xlog.Error("error loading config files", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
|
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
|
||||||
log.Error().Err(err).Msg("error registering external backends")
|
xlog.Error("error registering external backends", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.ConfigFile != "" {
|
if options.ConfigFile != "" {
|
||||||
if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
|
if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
|
||||||
log.Error().Err(err).Msg("error loading config file")
|
xlog.Error("error loading config file", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
|
if err := application.ModelConfigLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
|
||||||
log.Error().Err(err).Msg("error downloading models")
|
xlog.Error("error downloading models", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.PreloadJSONModels != "" {
|
if options.PreloadJSONModels != "" {
|
||||||
@@ -114,7 +114,7 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
|
|
||||||
if options.Debug {
|
if options.Debug {
|
||||||
for _, v := range application.ModelConfigLoader().GetAllModelsConfigs() {
|
for _, v := range application.ModelConfigLoader().GetAllModelsConfigs() {
|
||||||
log.Debug().Msgf("Model: %s (config: %+v)", v.Name, v)
|
xlog.Debug("Model", "name", v.Name, "config", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,10 +128,10 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
// turn off any process that was started by GRPC if the context is canceled
|
// turn off any process that was started by GRPC if the context is canceled
|
||||||
go func() {
|
go func() {
|
||||||
<-options.Context.Done()
|
<-options.Context.Done()
|
||||||
log.Debug().Msgf("Context canceled, shutting down")
|
xlog.Debug("Context canceled, shutting down")
|
||||||
err := application.ModelLoader().StopAllGRPC()
|
err := application.ModelLoader().StopAllGRPC()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error while stopping all grpc backends")
|
xlog.Error("error while stopping all grpc backends", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -145,7 +145,7 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Auto loading model %s into memory from file: %s", m, cfg.Model)
|
xlog.Debug("Auto loading model into memory from file", "model", m, "file", cfg.Model)
|
||||||
|
|
||||||
o := backend.ModelOptions(*cfg, options)
|
o := backend.ModelOptions(*cfg, options)
|
||||||
|
|
||||||
@@ -160,7 +160,7 @@ func New(opts ...config.AppOption) (*Application, error) {
|
|||||||
// Watch the configuration directory
|
// Watch the configuration directory
|
||||||
startWatcher(options)
|
startWatcher(options)
|
||||||
|
|
||||||
log.Info().Msg("core/startup process completed!")
|
xlog.Info("core/startup process completed!")
|
||||||
return application, nil
|
return application, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,18 +174,18 @@ func startWatcher(options *config.ApplicationConfig) {
|
|||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
// We try to create the directory if it does not exist and was specified
|
// We try to create the directory if it does not exist and was specified
|
||||||
if err := os.MkdirAll(options.DynamicConfigsDir, 0700); err != nil {
|
if err := os.MkdirAll(options.DynamicConfigsDir, 0700); err != nil {
|
||||||
log.Error().Err(err).Msg("failed creating DynamicConfigsDir")
|
xlog.Error("failed creating DynamicConfigsDir", "error", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// something else happened, we log the error and don't start the watcher
|
// something else happened, we log the error and don't start the watcher
|
||||||
log.Error().Err(err).Msg("failed to read DynamicConfigsDir, watcher will not be started")
|
xlog.Error("failed to read DynamicConfigsDir, watcher will not be started", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
configHandler := newConfigFileHandler(options)
|
configHandler := newConfigFileHandler(options)
|
||||||
if err := configHandler.Watch(); err != nil {
|
if err := configHandler.Watch(); err != nil {
|
||||||
log.Error().Err(err).Msg("failed creating watcher")
|
xlog.Error("failed creating watcher", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,17 +211,17 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
|
|||||||
fileContent, err := os.ReadFile(settingsFile)
|
fileContent, err := os.ReadFile(settingsFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
log.Debug().Msg("runtime_settings.json not found, using defaults")
|
xlog.Debug("runtime_settings.json not found, using defaults")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Warn().Err(err).Msg("failed to read runtime_settings.json")
|
xlog.Warn("failed to read runtime_settings.json", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
var settings config.RuntimeSettings
|
var settings config.RuntimeSettings
|
||||||
|
|
||||||
if err := json.Unmarshal(fileContent, &settings); err != nil {
|
if err := json.Unmarshal(fileContent, &settings); err != nil {
|
||||||
log.Warn().Err(err).Msg("failed to parse runtime_settings.json")
|
xlog.Warn("failed to parse runtime_settings.json", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -257,7 +257,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
options.WatchDogIdleTimeout = dur
|
options.WatchDogIdleTimeout = dur
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
|
xlog.Warn("invalid watchdog idle timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogIdleTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -267,7 +267,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
options.WatchDogBusyTimeout = dur
|
options.WatchDogBusyTimeout = dur
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
|
xlog.Warn("invalid watchdog busy timeout in runtime_settings.json", "error", err, "timeout", *settings.WatchdogBusyTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -277,7 +277,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
options.WatchDogInterval = dur
|
options.WatchDogInterval = dur
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("interval", *settings.WatchdogInterval).Msg("invalid watchdog interval in runtime_settings.json")
|
xlog.Warn("invalid watchdog interval in runtime_settings.json", "error", err, "interval", *settings.WatchdogInterval)
|
||||||
options.WatchDogInterval = model.DefaultWatchdogInterval
|
options.WatchDogInterval = model.DefaultWatchdogInterval
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -331,7 +331,7 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msg("Runtime settings loaded from runtime_settings.json")
|
xlog.Debug("Runtime settings loaded from runtime_settings.json")
|
||||||
}
|
}
|
||||||
|
|
||||||
// initializeWatchdog initializes the watchdog with current ApplicationConfig settings
|
// initializeWatchdog initializes the watchdog with current ApplicationConfig settings
|
||||||
@@ -362,7 +362,7 @@ func initializeWatchdog(application *Application, options *config.ApplicationCon
|
|||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
<-options.Context.Done()
|
<-options.Context.Done()
|
||||||
log.Debug().Msgf("Context canceled, shutting down")
|
xlog.Debug("Context canceled, shutting down")
|
||||||
wd.Shutdown()
|
wd.Shutdown()
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *Application) StopWatchdog() error {
|
func (a *Application) StopWatchdog() error {
|
||||||
@@ -52,24 +52,17 @@ func (a *Application) startWatchdog() error {
|
|||||||
go func() {
|
go func() {
|
||||||
select {
|
select {
|
||||||
case <-a.watchdogStop:
|
case <-a.watchdogStop:
|
||||||
log.Debug().Msg("Watchdog stop signal received")
|
xlog.Debug("Watchdog stop signal received")
|
||||||
wd.Shutdown()
|
wd.Shutdown()
|
||||||
case <-appConfig.Context.Done():
|
case <-appConfig.Context.Done():
|
||||||
log.Debug().Msg("Context canceled, shutting down watchdog")
|
xlog.Debug("Context canceled, shutting down watchdog")
|
||||||
wd.Shutdown()
|
wd.Shutdown()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
log.Info().
|
xlog.Info("Watchdog started with new settings", "lruLimit", lruLimit, "busyCheck", appConfig.WatchDogBusy, "idleCheck", appConfig.WatchDogIdle, "memoryReclaimer", appConfig.MemoryReclaimerEnabled, "memoryThreshold", appConfig.MemoryReclaimerThreshold, "interval", appConfig.WatchDogInterval)
|
||||||
Int("lruLimit", lruLimit).
|
|
||||||
Bool("busyCheck", appConfig.WatchDogBusy).
|
|
||||||
Bool("idleCheck", appConfig.WatchDogIdle).
|
|
||||||
Bool("memoryReclaimer", appConfig.MemoryReclaimerEnabled).
|
|
||||||
Float64("memoryThreshold", appConfig.MemoryReclaimerThreshold).
|
|
||||||
Dur("interval", appConfig.WatchDogInterval).
|
|
||||||
Msg("Watchdog started with new settings")
|
|
||||||
} else {
|
} else {
|
||||||
log.Info().Msg("Watchdog disabled")
|
xlog.Info("Watchdog disabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
@@ -49,7 +49,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
|
|||||||
// if we failed to load the model, we try to download it
|
// if we failed to load the model, we try to download it
|
||||||
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
|
err := gallery.InstallModelFromGallery(ctx, o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile)
|
xlog.Error("failed to install model from gallery", "error", err, "model", modelFile)
|
||||||
//return nil, err
|
//return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -225,7 +225,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
|
|||||||
if !ok {
|
if !ok {
|
||||||
r, err := regexp.Compile(c)
|
r, err := regexp.Compile(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Err(err).Msg("failed to compile regex")
|
xlog.Fatal("failed to compile regex", "error", err)
|
||||||
}
|
}
|
||||||
cutstrings[c] = r
|
cutstrings[c] = r
|
||||||
reg = cutstrings[c]
|
reg = cutstrings[c]
|
||||||
@@ -242,7 +242,7 @@ func Finetune(config config.ModelConfig, input, prediction string) string {
|
|||||||
if !ok {
|
if !ok {
|
||||||
regex, err := regexp.Compile(r)
|
regex, err := regexp.Compile(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal().Err(err).Msg("failed to compile regex")
|
xlog.Fatal("failed to compile regex", "error", err)
|
||||||
}
|
}
|
||||||
cutstrings[r] = regex
|
cutstrings[r] = regex
|
||||||
reg = regex
|
reg = regex
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
|
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
|
||||||
@@ -208,7 +208,7 @@ func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
promptCachePath = p
|
promptCachePath = p
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Str("promptCachePath", promptCachePath).Msg("error creating prompt cache folder")
|
xlog.Error("error creating prompt cache folder", "error", err, "promptCachePath", promptCachePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -49,7 +49,7 @@ type BackendsCMD struct {
|
|||||||
func (bl *BackendsList) Run(ctx *cliContext.Context) error {
|
func (bl *BackendsList) Run(ctx *cliContext.Context) error {
|
||||||
var galleries []config.Gallery
|
var galleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(bl.BackendGalleries), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(bl.BackendGalleries), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load galleries")
|
xlog.Error("unable to load galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
systemState, err := system.GetSystemState(
|
systemState, err := system.GetSystemState(
|
||||||
@@ -77,7 +77,7 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error {
|
|||||||
func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
|
func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
|
||||||
var galleries []config.Gallery
|
var galleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(bi.BackendGalleries), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(bi.BackendGalleries), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load galleries")
|
xlog.Error("unable to load galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
systemState, err := system.GetSystemState(
|
systemState, err := system.GetSystemState(
|
||||||
@@ -98,7 +98,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
v := int(percentage * 10)
|
v := int(percentage * 10)
|
||||||
err := progressBar.Set(v)
|
err := progressBar.Set(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar")
|
xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +113,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
|
|
||||||
func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
|
func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
|
||||||
for _, backendName := range bu.BackendArgs {
|
for _, backendName := range bu.BackendArgs {
|
||||||
log.Info().Str("backend", backendName).Msg("uninstalling backend")
|
xlog.Info("uninstalling backend", "backend", backendName)
|
||||||
|
|
||||||
systemState, err := system.GetSystemState(
|
systemState, err := system.GetSystemState(
|
||||||
system.WithBackendSystemPath(bu.BackendsSystemPath),
|
system.WithBackendSystemPath(bu.BackendsSystemPath),
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/explorer"
|
"github.com/mudler/LocalAI/core/explorer"
|
||||||
"github.com/mudler/LocalAI/core/http"
|
"github.com/mudler/LocalAI/core/http"
|
||||||
"github.com/mudler/LocalAI/pkg/signals"
|
"github.com/mudler/LocalAI/pkg/signals"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ExplorerCMD struct {
|
type ExplorerCMD struct {
|
||||||
@@ -51,7 +51,7 @@ func (e *ExplorerCMD) Run(ctx *cliContext.Context) error {
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if err := appHTTP.Shutdown(ctx); err != nil {
|
if err := appHTTP.Shutdown(ctx); err != nil {
|
||||||
log.Error().Err(err).Msg("error during shutdown")
|
xlog.Error("error during shutdown", "error", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/downloader"
|
"github.com/mudler/LocalAI/pkg/downloader"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"github.com/schollz/progressbar/v3"
|
"github.com/schollz/progressbar/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ type ModelsCMD struct {
|
|||||||
func (ml *ModelsList) Run(ctx *cliContext.Context) error {
|
func (ml *ModelsList) Run(ctx *cliContext.Context) error {
|
||||||
var galleries []config.Gallery
|
var galleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load galleries")
|
xlog.Error("unable to load galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
systemState, err := system.GetSystemState(
|
systemState, err := system.GetSystemState(
|
||||||
@@ -88,12 +88,12 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
|
|
||||||
var galleries []config.Gallery
|
var galleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load galleries")
|
xlog.Error("unable to load galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var backendGalleries []config.Gallery
|
var backendGalleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(mi.BackendGalleries), &backendGalleries); err != nil {
|
if err := json.Unmarshal([]byte(mi.BackendGalleries), &backendGalleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load backend galleries")
|
xlog.Error("unable to load backend galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, modelName := range mi.ModelArgs {
|
for _, modelName := range mi.ModelArgs {
|
||||||
@@ -108,7 +108,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
v := int(percentage * 10)
|
v := int(percentage * 10)
|
||||||
err := progressBar.Set(v)
|
err := progressBar.Set(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar")
|
xlog.Error("error while updating progress bar", "error", err, "filename", fileName, "value", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//startup.InstallModels()
|
//startup.InstallModels()
|
||||||
@@ -122,7 +122,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
if !modelURI.LooksLikeOCI() {
|
if !modelURI.LooksLikeOCI() {
|
||||||
model := gallery.FindGalleryElement(models, modelName)
|
model := gallery.FindGalleryElement(models, modelName)
|
||||||
if model == nil {
|
if model == nil {
|
||||||
log.Error().Str("model", modelName).Msg("model not found")
|
xlog.Error("model not found", "model", modelName)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -131,7 +131,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model")
|
xlog.Info("installing model", "model", modelName, "license", model.License)
|
||||||
}
|
}
|
||||||
|
|
||||||
modelLoader := model.NewModelLoader(systemState)
|
modelLoader := model.NewModelLoader(systemState)
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/internal"
|
"github.com/mudler/LocalAI/internal"
|
||||||
"github.com/mudler/LocalAI/pkg/signals"
|
"github.com/mudler/LocalAI/pkg/signals"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog"
|
"github.com/mudler/xlog"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type RunCMD struct {
|
type RunCMD struct {
|
||||||
@@ -108,7 +107,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
|||||||
config.WithYAMLConfigPreload(r.PreloadModelsConfig),
|
config.WithYAMLConfigPreload(r.PreloadModelsConfig),
|
||||||
config.WithSystemState(systemState),
|
config.WithSystemState(systemState),
|
||||||
config.WithContextSize(r.ContextSize),
|
config.WithContextSize(r.ContextSize),
|
||||||
config.WithDebug(zerolog.GlobalLevel() <= zerolog.DebugLevel),
|
config.WithDebug(ctx.Debug || (ctx.LogLevel != nil && *ctx.LogLevel == "debug")),
|
||||||
config.WithGeneratedContentDir(r.GeneratedContentPath),
|
config.WithGeneratedContentDir(r.GeneratedContentPath),
|
||||||
config.WithUploadDir(r.UploadPath),
|
config.WithUploadDir(r.UploadPath),
|
||||||
config.WithDynamicConfigDir(r.LocalaiConfigDir),
|
config.WithDynamicConfigDir(r.LocalaiConfigDir),
|
||||||
@@ -138,7 +137,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
|||||||
tunnelEnvVar := strings.Join(tunnels, ",")
|
tunnelEnvVar := strings.Join(tunnels, ",")
|
||||||
// TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable
|
// TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable
|
||||||
os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
|
os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
|
||||||
log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar)
|
xlog.Debug("setting LLAMACPP_GRPC_SERVERS", "value", tunnelEnvVar)
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,17 +151,17 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
|||||||
|
|
||||||
token := ""
|
token := ""
|
||||||
if r.Peer2Peer || r.Peer2PeerToken != "" {
|
if r.Peer2Peer || r.Peer2PeerToken != "" {
|
||||||
log.Info().Msg("P2P mode enabled")
|
xlog.Info("P2P mode enabled")
|
||||||
token = r.Peer2PeerToken
|
token = r.Peer2PeerToken
|
||||||
if token == "" {
|
if token == "" {
|
||||||
// IF no token is provided, and p2p is enabled,
|
// IF no token is provided, and p2p is enabled,
|
||||||
// we generate one and wait for the user to pick up the token (this is for interactive)
|
// we generate one and wait for the user to pick up the token (this is for interactive)
|
||||||
log.Info().Msg("No token provided, generating one")
|
xlog.Info("No token provided, generating one")
|
||||||
token = p2p.GenerateToken(r.Peer2PeerDHTInterval, r.Peer2PeerOTPInterval)
|
token = p2p.GenerateToken(r.Peer2PeerDHTInterval, r.Peer2PeerOTPInterval)
|
||||||
log.Info().Msg("Generated Token:")
|
xlog.Info("Generated Token:")
|
||||||
fmt.Println(token)
|
fmt.Println(token)
|
||||||
|
|
||||||
log.Info().Msg("To use the token, you can run the following command in another node or terminal:")
|
xlog.Info("To use the token, you can run the following command in another node or terminal:")
|
||||||
fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token)
|
fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token)
|
||||||
}
|
}
|
||||||
opts = append(opts, config.WithP2PToken(token))
|
opts = append(opts, config.WithP2PToken(token))
|
||||||
@@ -248,7 +247,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
|||||||
|
|
||||||
appHTTP, err := http.API(app)
|
appHTTP, err := http.API(app)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error during HTTP App construction")
|
xlog.Error("error during HTTP App construction", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -260,7 +259,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
|
|||||||
|
|
||||||
signals.RegisterGracefulTerminationHandler(func() {
|
signals.RegisterGracefulTerminationHandler(func() {
|
||||||
if err := app.ModelLoader().StopAllGRPC(); err != nil {
|
if err := app.ModelLoader().StopAllGRPC(); err != nil {
|
||||||
log.Error().Err(err).Msg("error while stopping all grpc backends")
|
xlog.Error("error while stopping all grpc backends", "error", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SoundGenerationCMD struct {
|
type SoundGenerationCMD struct {
|
||||||
@@ -84,7 +84,7 @@ func (t *SoundGenerationCMD) Run(ctx *cliContext.Context) error {
|
|||||||
defer func() {
|
defer func() {
|
||||||
err := ml.StopAllGRPC()
|
err := ml.StopAllGRPC()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("unable to stop all grpc processes")
|
xlog.Error("unable to stop all grpc processes", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TranscriptCMD struct {
|
type TranscriptCMD struct {
|
||||||
@@ -54,7 +54,7 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
|
|||||||
defer func() {
|
defer func() {
|
||||||
err := ml.StopAllGRPC()
|
err := ml.StopAllGRPC()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("unable to stop all grpc processes")
|
xlog.Error("unable to stop all grpc processes", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TTSCMD struct {
|
type TTSCMD struct {
|
||||||
@@ -53,7 +53,7 @@ func (t *TTSCMD) Run(ctx *cliContext.Context) error {
|
|||||||
defer func() {
|
defer func() {
|
||||||
err := ml.StopAllGRPC()
|
err := ml.StopAllGRPC()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("unable to stop all grpc processes")
|
xlog.Error("unable to stop all grpc processes", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/mholt/archiver/v3"
|
"github.com/mholt/archiver/v3"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
gguf "github.com/gpustack/gguf-parser-go"
|
gguf "github.com/gpustack/gguf-parser-go"
|
||||||
cliContext "github.com/mudler/LocalAI/core/cli/context"
|
cliContext "github.com/mudler/LocalAI/core/cli/context"
|
||||||
@@ -51,7 +51,7 @@ type CreateOCIImageCMD struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
|
func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
|
||||||
log.Info().Msg("Creating OCI image from input")
|
xlog.Info("Creating OCI image from input")
|
||||||
|
|
||||||
dir, err := os.MkdirTemp("", "localai")
|
dir, err := os.MkdirTemp("", "localai")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -62,7 +62,7 @@ func (u *CreateOCIImageCMD) Run(ctx *cliContext.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Info().Msgf("Creating '%s' as '%s' from %v", u.Output, u.Input, u.Input)
|
xlog.Info("Creating OCI image", "output", u.Output, "input", u.Input)
|
||||||
|
|
||||||
platform := strings.Split(u.Platform, "/")
|
platform := strings.Split(u.Platform, "/")
|
||||||
if len(platform) != 2 {
|
if len(platform) != 2 {
|
||||||
@@ -80,27 +80,23 @@ func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error {
|
|||||||
f, err := gguf.ParseGGUFFile(u.Args[0])
|
f, err := gguf.ParseGGUFFile(u.Args[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Only valid for gguf files
|
// Only valid for gguf files
|
||||||
log.Error().Msgf("guessDefaultsFromFile: %s", "not a GGUF file")
|
xlog.Error("guessDefaultsFromFile: not a GGUF file")
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().
|
xlog.Info("GGUF file loaded", "file", u.Args[0], "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
|
||||||
Any("eosTokenID", f.Tokenizer().EOSTokenID).
|
|
||||||
Any("bosTokenID", f.Tokenizer().BOSTokenID).
|
|
||||||
Any("modelName", f.Metadata().Name).
|
|
||||||
Any("architecture", f.Architecture().Architecture).Msgf("GGUF file loaded: %s", u.Args[0])
|
|
||||||
|
|
||||||
log.Info().Any("tokenizer", fmt.Sprintf("%+v", f.Tokenizer())).Msg("Tokenizer")
|
xlog.Info("Tokenizer", "tokenizer", fmt.Sprintf("%+v", f.Tokenizer()))
|
||||||
log.Info().Any("architecture", fmt.Sprintf("%+v", f.Architecture())).Msg("Architecture")
|
xlog.Info("Architecture", "architecture", fmt.Sprintf("%+v", f.Architecture()))
|
||||||
|
|
||||||
v, exists := f.Header.MetadataKV.Get("tokenizer.chat_template")
|
v, exists := f.Header.MetadataKV.Get("tokenizer.chat_template")
|
||||||
if exists {
|
if exists {
|
||||||
log.Info().Msgf("chat_template: %s", v.ValueString())
|
xlog.Info("chat_template", "template", v.ValueString())
|
||||||
}
|
}
|
||||||
|
|
||||||
if u.Header {
|
if u.Header {
|
||||||
for _, metadata := range f.Header.MetadataKV {
|
for _, metadata := range f.Header.MetadataKV {
|
||||||
log.Info().Msgf("%s: %+v", metadata.Key, metadata.Value)
|
xlog.Info("metadata", "key", metadata.Key, "value", metadata.Value)
|
||||||
}
|
}
|
||||||
// log.Info().Any("header", fmt.Sprintf("%+v", f.Header)).Msg("Header")
|
// log.Info().Any("header", fmt.Sprintf("%+v", f.Header)).Msg("Header")
|
||||||
}
|
}
|
||||||
@@ -117,63 +113,63 @@ func (hfscmd *HFScanCMD) Run(ctx *cliContext.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Msg("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!")
|
xlog.Info("LocalAI Security Scanner - This is BEST EFFORT functionality! Currently limited to huggingface models!")
|
||||||
if len(hfscmd.ToScan) == 0 {
|
if len(hfscmd.ToScan) == 0 {
|
||||||
log.Info().Msg("Checking all installed models against galleries")
|
xlog.Info("Checking all installed models against galleries")
|
||||||
var galleries []config.Gallery
|
var galleries []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(hfscmd.Galleries), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(hfscmd.Galleries), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("unable to load galleries")
|
xlog.Error("unable to load galleries", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := gallery.SafetyScanGalleryModels(galleries, systemState)
|
err := gallery.SafetyScanGalleryModels(galleries, systemState)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
|
xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Msg("! WARNING ! A known-vulnerable model is installed!")
|
xlog.Error("! WARNING ! A known-vulnerable model is installed!", "error", err)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
} else {
|
} else {
|
||||||
var errs error = nil
|
var errs error = nil
|
||||||
for _, uri := range hfscmd.ToScan {
|
for _, uri := range hfscmd.ToScan {
|
||||||
log.Info().Str("uri", uri).Msg("scanning specific uri")
|
xlog.Info("scanning specific uri", "uri", uri)
|
||||||
scanResults, err := downloader.HuggingFaceScan(downloader.URI(uri))
|
scanResults, err := downloader.HuggingFaceScan(downloader.URI(uri))
|
||||||
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||||
log.Error().Err(err).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("! WARNING ! A known-vulnerable model is included in this repo!")
|
xlog.Error("! WARNING ! A known-vulnerable model is included in this repo!", "error", err, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
|
||||||
errs = errors.Join(errs, err)
|
errs = errors.Join(errs, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errs != nil {
|
if errs != nil {
|
||||||
return errs
|
return errs
|
||||||
}
|
}
|
||||||
log.Info().Msg("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
|
xlog.Info("No security warnings were detected for your installed models. Please note that this is a BEST EFFORT tool, and all issues may not be detected.")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error {
|
func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error {
|
||||||
if len(uhcmd.ConfigName) == 0 {
|
if len(uhcmd.ConfigName) == 0 {
|
||||||
log.Error().Msg("ConfigName is a required parameter")
|
xlog.Error("ConfigName is a required parameter")
|
||||||
return fmt.Errorf("config name is a required parameter")
|
return fmt.Errorf("config name is a required parameter")
|
||||||
}
|
}
|
||||||
if len(uhcmd.ModelsPath) == 0 {
|
if len(uhcmd.ModelsPath) == 0 {
|
||||||
log.Error().Msg("ModelsPath is a required parameter")
|
xlog.Error("ModelsPath is a required parameter")
|
||||||
return fmt.Errorf("model path is a required parameter")
|
return fmt.Errorf("model path is a required parameter")
|
||||||
}
|
}
|
||||||
bcl := config.NewModelConfigLoader(uhcmd.ModelsPath)
|
bcl := config.NewModelConfigLoader(uhcmd.ModelsPath)
|
||||||
err := bcl.ReadModelConfig(uhcmd.ConfigName)
|
err := bcl.ReadModelConfig(uhcmd.ConfigName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("ConfigName", uhcmd.ConfigName).Msg("error while loading backend")
|
xlog.Error("error while loading backend", "error", err, "ConfigName", uhcmd.ConfigName)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bc, exists := bcl.GetModelConfig(uhcmd.ConfigName)
|
bc, exists := bcl.GetModelConfig(uhcmd.ConfigName)
|
||||||
if !exists {
|
if !exists {
|
||||||
log.Error().Str("ConfigName", uhcmd.ConfigName).Msg("ConfigName not found")
|
xlog.Error("ConfigName not found", "ConfigName", uhcmd.ConfigName)
|
||||||
}
|
}
|
||||||
for name, uc := range config.GetAllModelConfigUsecases() {
|
for name, uc := range config.GetAllModelConfigUsecases() {
|
||||||
if bc.HasUsecases(uc) {
|
if bc.HasUsecases(uc) {
|
||||||
log.Info().Str("Usecase", name)
|
xlog.Info("Usecase", "usecase", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Info().Msg("---")
|
xlog.Info("---")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/gallery"
|
"github.com/mudler/LocalAI/core/gallery"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type LLamaCPP struct {
|
type LLamaCPP struct {
|
||||||
@@ -30,22 +30,22 @@ const (
|
|||||||
func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) {
|
func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) {
|
||||||
backends, err := gallery.ListSystemBackends(systemState)
|
backends, err := gallery.ListSystemBackends(systemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Msgf("Failed listing system backends: %s", err)
|
xlog.Warn("Failed listing system backends", "error", err)
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("System backends: %v", backends)
|
xlog.Debug("System backends", "backends", backends)
|
||||||
|
|
||||||
backend, ok := backends.Get(llamaCPPGalleryName)
|
backend, ok := backends.Get(llamaCPPGalleryName)
|
||||||
if !ok {
|
if !ok {
|
||||||
ml := model.NewModelLoader(systemState)
|
ml := model.NewModelLoader(systemState)
|
||||||
var gals []config.Gallery
|
var gals []config.Gallery
|
||||||
if err := json.Unmarshal([]byte(galleries), &gals); err != nil {
|
if err := json.Unmarshal([]byte(galleries), &gals); err != nil {
|
||||||
log.Error().Err(err).Msg("failed loading galleries")
|
xlog.Error("failed loading galleries", "error", err)
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
err := gallery.InstallBackendFromGallery(context.Background(), gals, systemState, ml, llamaCPPGalleryName, nil, true)
|
err := gallery.InstallBackendFromGallery(context.Background(), gals, systemState, ml, llamaCPPGalleryName, nil, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("llama-cpp backend not found, failed to install it")
|
xlog.Error("llama-cpp backend not found, failed to install it", "error", err)
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/p2p"
|
"github.com/mudler/LocalAI/core/p2p"
|
||||||
"github.com/mudler/LocalAI/pkg/signals"
|
"github.com/mudler/LocalAI/pkg/signals"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
|
"github.com/mudler/xlog"
|
||||||
"github.com/phayes/freeport"
|
"github.com/phayes/freeport"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type P2P struct {
|
type P2P struct {
|
||||||
@@ -66,16 +66,16 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Info().Msgf("You need to start llama-cpp-rpc-server on '%s:%s'", address, p)
|
xlog.Info("You need to start llama-cpp-rpc-server", "address", address, "port", p)
|
||||||
} else {
|
} else {
|
||||||
// Start llama.cpp directly from the version we have pre-packaged
|
// Start llama.cpp directly from the version we have pre-packaged
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
log.Info().Msgf("Starting llama-cpp-rpc-server on '%s:%d'", address, port)
|
xlog.Info("Starting llama-cpp-rpc-server", "address", address, "port", port)
|
||||||
|
|
||||||
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
|
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to find llama-cpp-rpc-server")
|
xlog.Error("Failed to find llama-cpp-rpc-server", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,7 +85,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
|
|||||||
extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ")
|
extraArgs = strings.Split(r.ExtraLLamaCPPArgs, " ")
|
||||||
}
|
}
|
||||||
args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...)
|
args := append([]string{"--host", address, "--port", fmt.Sprint(port)}, extraArgs...)
|
||||||
log.Debug().Msgf("Starting llama-cpp-rpc-server on '%s:%d' with args: %+v (%d)", address, port, args, len(args))
|
xlog.Debug("Starting llama-cpp-rpc-server", "address", address, "port", port, "args", args, "argCount", len(args))
|
||||||
|
|
||||||
cmd := exec.Command(
|
cmd := exec.Command(
|
||||||
grpcProcess, args...,
|
grpcProcess, args...,
|
||||||
@@ -97,7 +97,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
|
|||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
|
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
log.Error().Any("grpcProcess", grpcProcess).Any("args", args).Err(err).Msg("Failed to start llama-cpp-rpc-server")
|
xlog.Error("Failed to start llama-cpp-rpc-server", "error", err, "grpcProcess", grpcProcess, "args", args)
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.Wait()
|
cmd.Wait()
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ApplicationConfig struct {
|
type ApplicationConfig struct {
|
||||||
@@ -298,7 +298,7 @@ func WithStringGalleries(galls string) AppOption {
|
|||||||
}
|
}
|
||||||
var galleries []Gallery
|
var galleries []Gallery
|
||||||
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("failed loading galleries")
|
xlog.Error("failed loading galleries", "error", err)
|
||||||
}
|
}
|
||||||
o.Galleries = append(o.Galleries, galleries...)
|
o.Galleries = append(o.Galleries, galleries...)
|
||||||
}
|
}
|
||||||
@@ -312,7 +312,7 @@ func WithBackendGalleries(galls string) AppOption {
|
|||||||
}
|
}
|
||||||
var galleries []Gallery
|
var galleries []Gallery
|
||||||
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
|
if err := json.Unmarshal([]byte(galls), &galleries); err != nil {
|
||||||
log.Error().Err(err).Msg("failed loading galleries")
|
xlog.Error("failed loading galleries", "error", err)
|
||||||
}
|
}
|
||||||
o.BackendGalleries = append(o.BackendGalleries, galleries...)
|
o.BackendGalleries = append(o.BackendGalleries, galleries...)
|
||||||
}
|
}
|
||||||
@@ -470,7 +470,7 @@ func WithHttpGetExemptedEndpoints(endpoints []string) AppOption {
|
|||||||
if err == nil && r != nil {
|
if err == nil && r != nil {
|
||||||
o.HttpGetExemptedEndpoints = append(o.HttpGetExemptedEndpoints, r)
|
o.HttpGetExemptedEndpoints = append(o.HttpGetExemptedEndpoints, r)
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Err(err).Str("regex", epr).Msg("Error while compiling HTTP Get Exemption regex, skipping this entry.")
|
xlog.Warn("Error while compiling HTTP Get Exemption regex, skipping this entry.", "error", err, "regex", epr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
gguf "github.com/gpustack/gguf-parser-go"
|
gguf "github.com/gpustack/gguf-parser-go"
|
||||||
)
|
)
|
||||||
@@ -35,22 +35,22 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
|
|||||||
// vram estimation
|
// vram estimation
|
||||||
vram, err := xsysinfo.TotalAvailableVRAM()
|
vram, err := xsysinfo.TotalAvailableVRAM()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("guessDefaultsFromFile(TotalAvailableVRAM): %s", err)
|
xlog.Error("guessDefaultsFromFile(TotalAvailableVRAM)", "error", err)
|
||||||
} else if vram > 0 {
|
} else if vram > 0 {
|
||||||
estimate, err := xsysinfo.EstimateGGUFVRAMUsage(f, vram)
|
estimate, err := xsysinfo.EstimateGGUFVRAMUsage(f, vram)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("guessDefaultsFromFile(EstimateGGUFVRAMUsage): %s", err)
|
xlog.Error("guessDefaultsFromFile(EstimateGGUFVRAMUsage)", "error", err)
|
||||||
} else {
|
} else {
|
||||||
if estimate.IsFullOffload {
|
if estimate.IsFullOffload {
|
||||||
log.Warn().Msgf("guessDefaultsFromFile: %s", "full offload is recommended")
|
xlog.Warn("guessDefaultsFromFile: full offload is recommended")
|
||||||
}
|
}
|
||||||
|
|
||||||
if estimate.EstimatedVRAM > vram {
|
if estimate.EstimatedVRAM > vram {
|
||||||
log.Warn().Msgf("guessDefaultsFromFile: %s", "estimated VRAM usage is greater than available VRAM")
|
xlog.Warn("guessDefaultsFromFile: estimated VRAM usage is greater than available VRAM")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.NGPULayers == nil && estimate.EstimatedLayers > 0 {
|
if cfg.NGPULayers == nil && estimate.EstimatedLayers > 0 {
|
||||||
log.Debug().Msgf("guessDefaultsFromFile: %d layers estimated", estimate.EstimatedLayers)
|
xlog.Debug("guessDefaultsFromFile: layers estimated", "layers", estimate.EstimatedLayers)
|
||||||
cfg.NGPULayers = &estimate.EstimatedLayers
|
cfg.NGPULayers = &estimate.EstimatedLayers
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -62,20 +62,16 @@ func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
|
|||||||
cfg.NGPULayers = &defaultHigh
|
cfg.NGPULayers = &defaultHigh
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Any("NGPULayers", cfg.NGPULayers).Msgf("guessDefaultsFromFile: %s", "NGPULayers set")
|
xlog.Debug("guessDefaultsFromFile: NGPULayers set", "NGPULayers", cfg.NGPULayers)
|
||||||
|
|
||||||
// template estimations
|
// template estimations
|
||||||
if cfg.HasTemplate() {
|
if cfg.HasTemplate() {
|
||||||
// nothing to guess here
|
// nothing to guess here
|
||||||
log.Debug().Any("name", cfg.Name).Msgf("guessDefaultsFromFile: %s", "template already set")
|
xlog.Debug("guessDefaultsFromFile: template already set", "name", cfg.Name)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().
|
xlog.Debug("Model file loaded", "file", cfg.ModelFileName(), "eosTokenID", f.Tokenizer().EOSTokenID, "bosTokenID", f.Tokenizer().BOSTokenID, "modelName", f.Metadata().Name, "architecture", f.Architecture().Architecture)
|
||||||
Any("eosTokenID", f.Tokenizer().EOSTokenID).
|
|
||||||
Any("bosTokenID", f.Tokenizer().BOSTokenID).
|
|
||||||
Any("modelName", f.Metadata().Name).
|
|
||||||
Any("architecture", f.Architecture().Architecture).Msgf("Model file loaded: %s", cfg.ModelFileName())
|
|
||||||
|
|
||||||
// guess the name
|
// guess the name
|
||||||
if cfg.Name == "" {
|
if cfg.Name == "" {
|
||||||
|
|||||||
@@ -5,17 +5,17 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
gguf "github.com/gpustack/gguf-parser-go"
|
gguf "github.com/gpustack/gguf-parser-go"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
|
func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
|
||||||
if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
|
if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
|
||||||
log.Debug().Msgf("guessDefaultsFromFile: %s", "guessing disabled with LOCALAI_DISABLE_GUESSING")
|
xlog.Debug("guessDefaultsFromFile: guessing disabled with LOCALAI_DISABLE_GUESSING")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if modelPath == "" {
|
if modelPath == "" {
|
||||||
log.Debug().Msgf("guessDefaultsFromFile: %s", "modelPath is empty")
|
xlog.Debug("guessDefaultsFromFile: modelPath is empty")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if r := recover(); r != nil {
|
if r := recover(); r != nil {
|
||||||
log.Error().Msgf("guessDefaultsFromFile: %s", "panic while parsing gguf file")
|
xlog.Error("guessDefaultsFromFile: panic while parsing gguf file")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/downloader"
|
"github.com/mudler/LocalAI/pkg/downloader"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -250,7 +250,7 @@ func (bcl *ModelConfigLoader) Preload(modelPath string) error {
|
|||||||
utils.DisplayDownloadFunction(fileName, current, total, percent)
|
utils.DisplayDownloadFunction(fileName, current, total, percent)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Msgf("Preloading models from %s", modelPath)
|
xlog.Info("Preloading models", "path", modelPath)
|
||||||
|
|
||||||
renderMode := "dark"
|
renderMode := "dark"
|
||||||
if os.Getenv("COLOR") != "" {
|
if os.Getenv("COLOR") != "" {
|
||||||
@@ -270,7 +270,7 @@ func (bcl *ModelConfigLoader) Preload(modelPath string) error {
|
|||||||
|
|
||||||
// Download files and verify their SHA
|
// Download files and verify their SHA
|
||||||
for i, file := range config.DownloadFiles {
|
for i, file := range config.DownloadFiles {
|
||||||
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename)
|
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||||
|
|
||||||
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
|
if err := utils.VerifyPath(file.Filename, modelPath); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -361,13 +361,13 @@ func (bcl *ModelConfigLoader) LoadModelConfigsFromPath(path string, opts ...Conf
|
|||||||
}
|
}
|
||||||
c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...)
|
c, err := readModelConfigFromFile(filepath.Join(path, file.Name()), opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("File Name", file.Name()).Msgf("LoadModelConfigsFromPath cannot read config file")
|
xlog.Error("LoadModelConfigsFromPath cannot read config file", "error", err, "File Name", file.Name())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if valid, _ := c.Validate(); valid {
|
if valid, _ := c.Validate(); valid {
|
||||||
bcl.configs[c.Name] = *c
|
bcl.configs[c.Name] = *c
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Str("Name", c.Name).Msgf("config is not valid")
|
xlog.Error("config is not valid", "error", err, "Name", c.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/p2p"
|
"github.com/mudler/LocalAI/core/p2p"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
@@ -57,21 +57,21 @@ func (s *DiscoveryServer) runBackground() {
|
|||||||
// do not do in parallel
|
// do not do in parallel
|
||||||
n, err := p2p.NewNode(token)
|
n, err := p2p.NewNode(token)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Err(err).Msg("Failed to create node")
|
xlog.Error("Failed to create node", "error", err)
|
||||||
s.failedToken(token)
|
s.failedToken(token)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err = n.Start(c)
|
err = n.Start(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Err(err).Msg("Failed to start node")
|
xlog.Error("Failed to start node", "error", err)
|
||||||
s.failedToken(token)
|
s.failedToken(token)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
ledger, err := n.Ledger()
|
ledger, err := n.Ledger()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Err(err).Msg("Failed to start ledger")
|
xlog.Error("Failed to start ledger", "error", err)
|
||||||
s.failedToken(token)
|
s.failedToken(token)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -92,10 +92,10 @@ func (s *DiscoveryServer) runBackground() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Any("network", token).Msgf("Network has %d clusters", len(ledgerK))
|
xlog.Debug("Network clusters", "network", token, "count", len(ledgerK))
|
||||||
if len(ledgerK) != 0 {
|
if len(ledgerK) != 0 {
|
||||||
for _, k := range ledgerK {
|
for _, k := range ledgerK {
|
||||||
log.Debug().Any("network", token).Msgf("Clusterdata %+v", k)
|
xlog.Debug("Clusterdata", "network", token, "cluster", k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,7 +128,7 @@ func (s *DiscoveryServer) deleteFailedConnections() {
|
|||||||
for _, t := range s.database.TokenList() {
|
for _, t := range s.database.TokenList() {
|
||||||
data, _ := s.database.Get(t)
|
data, _ := s.database.Get(t)
|
||||||
if data.Failures > s.errorThreshold {
|
if data.Failures > s.errorThreshold {
|
||||||
log.Info().Any("token", t).Msg("Token has been removed from the database")
|
xlog.Info("Token has been removed from the database", "token", t)
|
||||||
s.database.Delete(t)
|
s.database.Delete(t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BackendMetadata represents the metadata stored in a JSON file for each installed backend
|
// BackendMetadata represents the metadata stored in a JSON file for each installed backend
|
||||||
@@ -37,11 +37,11 @@ func (backend *GalleryBackend) FindBestBackendFromMeta(systemState *system.Syste
|
|||||||
|
|
||||||
realBackend := backend.CapabilitiesMap[systemState.Capability(backend.CapabilitiesMap)]
|
realBackend := backend.CapabilitiesMap[systemState.Capability(backend.CapabilitiesMap)]
|
||||||
if realBackend == "" {
|
if realBackend == "" {
|
||||||
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("No backend found for reported capability")
|
xlog.Debug("No backend found for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("backend", backend.Name).Str("reportedCapability", systemState.Capability(backend.CapabilitiesMap)).Msg("Found backend for reported capability")
|
xlog.Debug("Found backend for reported capability", "backend", backend.Name, "reportedCapability", systemState.Capability(backend.CapabilitiesMap))
|
||||||
return backends.FindByName(realBackend)
|
return backends.FindByName(realBackend)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/downloader"
|
"github.com/mudler/LocalAI/pkg/downloader"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
|
"github.com/mudler/xlog"
|
||||||
cp "github.com/otiai10/copy"
|
cp "github.com/otiai10/copy"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -86,7 +86,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
|
|||||||
return fmt.Errorf("backend name is empty")
|
return fmt.Errorf("backend name is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Interface("galleries", galleries).Str("name", name).Msg("Installing backend from gallery")
|
xlog.Debug("Installing backend from gallery", "galleries", galleries, "name", name)
|
||||||
|
|
||||||
backends, err := AvailableBackends(galleries, systemState)
|
backends, err := AvailableBackends(galleries, systemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -99,7 +99,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if backend.IsMeta() {
|
if backend.IsMeta() {
|
||||||
log.Debug().Interface("systemState", systemState).Str("name", name).Msg("Backend is a meta backend")
|
xlog.Debug("Backend is a meta backend", "systemState", systemState, "name", name)
|
||||||
|
|
||||||
// Then, let's try to find the best backend based on the capabilities map
|
// Then, let's try to find the best backend based on the capabilities map
|
||||||
bestBackend := backend.FindBestBackendFromMeta(systemState, backends)
|
bestBackend := backend.FindBestBackendFromMeta(systemState, backends)
|
||||||
@@ -107,7 +107,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
|
|||||||
return fmt.Errorf("no backend found with capabilities %q", backend.CapabilitiesMap)
|
return fmt.Errorf("no backend found with capabilities %q", backend.CapabilitiesMap)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("name", name).Str("bestBackend", bestBackend.Name).Msg("Installing backend from meta backend")
|
xlog.Debug("Installing backend from meta backend", "name", name, "bestBackend", bestBackend.Name)
|
||||||
|
|
||||||
// Then, let's install the best backend
|
// Then, let's install the best backend
|
||||||
if err := InstallBackend(ctx, systemState, modelLoader, bestBackend, downloadStatus); err != nil {
|
if err := InstallBackend(ctx, systemState, modelLoader, bestBackend, downloadStatus); err != nil {
|
||||||
@@ -164,7 +164,7 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
|
|||||||
return fmt.Errorf("failed copying: %w", err)
|
return fmt.Errorf("failed copying: %w", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloading backend")
|
xlog.Debug("Downloading backend", "uri", config.URI, "backendPath", backendPath)
|
||||||
if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil {
|
if err := uri.DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err != nil {
|
||||||
success := false
|
success := false
|
||||||
// Try to download from mirrors
|
// Try to download from mirrors
|
||||||
@@ -177,24 +177,24 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
|
|||||||
}
|
}
|
||||||
if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
|
if err := downloader.URI(mirror).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil {
|
||||||
success = true
|
success = true
|
||||||
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend")
|
xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !success {
|
if !success {
|
||||||
log.Error().Str("uri", config.URI).Str("backendPath", backendPath).Err(err).Msg("Failed to download backend")
|
xlog.Error("Failed to download backend", "uri", config.URI, "backendPath", backendPath, "error", err)
|
||||||
return fmt.Errorf("failed to download backend %q: %v", config.URI, err)
|
return fmt.Errorf("failed to download backend %q: %v", config.URI, err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Str("uri", config.URI).Str("backendPath", backendPath).Msg("Downloaded backend")
|
xlog.Debug("Downloaded backend", "uri", config.URI, "backendPath", backendPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sanity check - check if runfile is present
|
// sanity check - check if runfile is present
|
||||||
runFile := filepath.Join(backendPath, runFile)
|
runFile := filepath.Join(backendPath, runFile)
|
||||||
if _, err := os.Stat(runFile); os.IsNotExist(err) {
|
if _, err := os.Stat(runFile); os.IsNotExist(err) {
|
||||||
log.Error().Str("runFile", runFile).Msg("Run file not found")
|
xlog.Error("Run file not found", "runFile", runFile)
|
||||||
return fmt.Errorf("not a valid backend: run file not found %q", runFile)
|
return fmt.Errorf("not a valid backend: run file not found %q", runFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,7 +271,7 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error
|
|||||||
|
|
||||||
if metadata != nil && metadata.MetaBackendFor != "" {
|
if metadata != nil && metadata.MetaBackendFor != "" {
|
||||||
metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor)
|
metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor)
|
||||||
log.Debug().Str("backendDirectory", metaBackendDirectory).Msg("Deleting meta backend")
|
xlog.Debug("Deleting meta backend", "backendDirectory", metaBackendDirectory)
|
||||||
if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) {
|
if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) {
|
||||||
return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor)
|
return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor)
|
||||||
}
|
}
|
||||||
@@ -330,9 +330,9 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
log.Warn().Err(err).Msg("Failed to read system backends, proceeding with user-managed backends")
|
xlog.Warn("Failed to read system backends, proceeding with user-managed backends", "error", err)
|
||||||
} else if errors.Is(err, os.ErrNotExist) {
|
} else if errors.Is(err, os.ErrNotExist) {
|
||||||
log.Debug().Msg("No system backends found")
|
xlog.Debug("No system backends found")
|
||||||
}
|
}
|
||||||
|
|
||||||
// User-managed backends and alias collection
|
// User-managed backends and alias collection
|
||||||
@@ -442,7 +442,7 @@ func RegisterBackends(systemState *system.SystemState, modelLoader *model.ModelL
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, backend := range backends {
|
for _, backend := range backends {
|
||||||
log.Debug().Str("name", backend.Name).Str("runFile", backend.RunFile).Msg("Registering backend")
|
xlog.Debug("Registering backend", "name", backend.Name, "runFile", backend.RunFile)
|
||||||
modelLoader.SetExternalBackend(backend.Name, backend.RunFile)
|
modelLoader.SetExternalBackend(backend.Name, backend.RunFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/downloader"
|
"github.com/mudler/LocalAI/pkg/downloader"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/xsync"
|
"github.com/mudler/LocalAI/pkg/xsync"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
@@ -26,7 +26,7 @@ func GetGalleryConfigFromURL[T any](url string, basePath string) (T, error) {
|
|||||||
return yaml.Unmarshal(d, &config)
|
return yaml.Unmarshal(d, &config)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
|
xlog.Error("failed to get gallery config for url", "error", err, "url", url)
|
||||||
return config, err
|
return config, err
|
||||||
}
|
}
|
||||||
return config, nil
|
return config, nil
|
||||||
@@ -39,7 +39,7 @@ func GetGalleryConfigFromURLWithContext[T any](ctx context.Context, url string,
|
|||||||
return yaml.Unmarshal(d, &config)
|
return yaml.Unmarshal(d, &config)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
|
xlog.Error("failed to get gallery config for url", "error", err, "url", url)
|
||||||
return config, err
|
return config, err
|
||||||
}
|
}
|
||||||
return config, nil
|
return config, nil
|
||||||
@@ -310,7 +310,7 @@ func getGalleryElements[T GalleryElement](gallery config.Gallery, basePath strin
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if yamlErr, ok := err.(*yaml.TypeError); ok {
|
if yamlErr, ok := err.(*yaml.TypeError); ok {
|
||||||
log.Debug().Msgf("YAML errors: %s\n\nwreckage of models: %+v", strings.Join(yamlErr.Errors, "\n"), models)
|
xlog.Debug("YAML errors", "errors", strings.Join(yamlErr.Errors, "\n"), "models", models)
|
||||||
}
|
}
|
||||||
return models, fmt.Errorf("failed to read gallery elements: %w", err)
|
return models, fmt.Errorf("failed to read gallery elements: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
@@ -52,10 +52,10 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// maybe not a HF repository
|
// maybe not a HF repository
|
||||||
// TODO: maybe we can check if the URI is a valid HF repository
|
// TODO: maybe we can check if the URI is a valid HF repository
|
||||||
log.Debug().Str("uri", uri).Str("hfrepoID", hfrepoID).Msg("Failed to get model details, maybe not a HF repository")
|
xlog.Debug("Failed to get model details, maybe not a HF repository", "uri", uri, "hfrepoID", hfrepoID)
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Str("uri", uri).Msg("Got model details")
|
xlog.Debug("Got model details", "uri", uri)
|
||||||
log.Debug().Any("details", hfDetails).Msg("Model details")
|
xlog.Debug("Model details", "details", hfDetails)
|
||||||
}
|
}
|
||||||
|
|
||||||
// handle local config files ("/my-model.yaml" or "file://my-model.yaml")
|
// handle local config files ("/my-model.yaml" or "file://my-model.yaml")
|
||||||
@@ -73,13 +73,13 @@ func DiscoverModelConfig(uri string, preferences json.RawMessage) (gallery.Model
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition")
|
xlog.Error("error reading model definition", "error", err, "filepath", localURI)
|
||||||
return gallery.ModelConfig{}, err
|
return gallery.ModelConfig{}, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
modelYAML, err = os.ReadFile(localURI)
|
modelYAML, err = os.ReadFile(localURI)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("filepath", localURI).Msg("error reading model definition")
|
xlog.Error("error reading model definition", "error", err, "filepath", localURI)
|
||||||
return gallery.ModelConfig{}, err
|
return gallery.ModelConfig{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/downloader"
|
"github.com/mudler/LocalAI/pkg/downloader"
|
||||||
"github.com/mudler/LocalAI/pkg/functions"
|
"github.com/mudler/LocalAI/pkg/functions"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"go.yaml.in/yaml/v2"
|
"go.yaml.in/yaml/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ type LlamaCPPImporter struct{}
|
|||||||
func (i *LlamaCPPImporter) Match(details Details) bool {
|
func (i *LlamaCPPImporter) Match(details Details) bool {
|
||||||
preferences, err := details.Preferences.MarshalJSON()
|
preferences, err := details.Preferences.MarshalJSON()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("failed to marshal preferences")
|
xlog.Error("failed to marshal preferences", "error", err)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,7 +31,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
|
|||||||
if len(preferences) > 0 {
|
if len(preferences) > 0 {
|
||||||
err = json.Unmarshal(preferences, &preferencesMap)
|
err = json.Unmarshal(preferences, &preferencesMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("failed to unmarshal preferences")
|
xlog.Error("failed to unmarshal preferences", "error", err)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -63,7 +63,7 @@ func (i *LlamaCPPImporter) Match(details Details) bool {
|
|||||||
|
|
||||||
func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) {
|
func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error) {
|
||||||
|
|
||||||
log.Debug().Str("uri", details.URI).Msg("llama.cpp importer matched")
|
xlog.Debug("llama.cpp importer matched", "uri", details.URI)
|
||||||
|
|
||||||
preferences, err := details.Preferences.MarshalJSON()
|
preferences, err := details.Preferences.MarshalJSON()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -131,9 +131,9 @@ func InstallModelFromGallery(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Installed model %q", installedModel.Name)
|
xlog.Debug("Installed model", "model", installedModel.Name)
|
||||||
if automaticallyInstallBackend && installedModel.Backend != "" {
|
if automaticallyInstallBackend && installedModel.Backend != "" {
|
||||||
log.Debug().Msgf("Installing backend %q", installedModel.Backend)
|
xlog.Debug("Installing backend", "backend", installedModel.Backend)
|
||||||
|
|
||||||
if err := InstallBackendFromGallery(ctx, backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil {
|
if err := InstallBackendFromGallery(ctx, backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -165,7 +165,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(configOverrides) > 0 {
|
if len(configOverrides) > 0 {
|
||||||
log.Debug().Msgf("Config overrides %+v", configOverrides)
|
xlog.Debug("Config overrides", "overrides", configOverrides)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download files and verify their SHA
|
// Download files and verify their SHA
|
||||||
@@ -177,7 +177,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Checking %q exists and matches SHA", file.Filename)
|
xlog.Debug("Checking file exists and matches SHA", "filename", file.Filename)
|
||||||
|
|
||||||
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
|
if err := utils.VerifyPath(file.Filename, basePath); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -189,7 +189,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
if enforceScan {
|
if enforceScan {
|
||||||
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
||||||
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||||
log.Error().Str("model", config.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
|
xlog.Error("Contains unsafe file(s)!", "model", config.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -218,7 +218,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
|
return nil, fmt.Errorf("failed to write prompt template %q: %v", template.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Prompt template %q written", template.Name)
|
xlog.Debug("Prompt template written", "template", template.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
name := config.Name
|
name := config.Name
|
||||||
@@ -269,7 +269,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
return nil, fmt.Errorf("failed to write updated config file: %v", err)
|
return nil, fmt.Errorf("failed to write updated config file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Written config file %s", configFilePath)
|
xlog.Debug("Written config file", "file", configFilePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save the model gallery file for further reference
|
// Save the model gallery file for further reference
|
||||||
@@ -279,7 +279,7 @@ func InstallModel(ctx context.Context, systemState *system.SystemState, nameOver
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Written gallery file %s", modelFile)
|
xlog.Debug("Written gallery file", "file", modelFile)
|
||||||
|
|
||||||
return &modelConfig, os.WriteFile(modelFile, data, 0600)
|
return &modelConfig, os.WriteFile(modelFile, data, 0600)
|
||||||
}
|
}
|
||||||
@@ -341,7 +341,7 @@ func listModelFiles(systemState *system.SystemState, name string) ([]string, err
|
|||||||
allFiles = append(allFiles, fullPath)
|
allFiles = append(allFiles, fullPath)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Msgf("failed to read gallery file %s", configFile)
|
xlog.Error("failed to read gallery file", "error", err, "file", configFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, f := range additionalFiles {
|
for _, f := range additionalFiles {
|
||||||
@@ -391,26 +391,26 @@ func DeleteModelFromSystem(systemState *system.SystemState, name string) error {
|
|||||||
name := strings.TrimSuffix(f.Name(), ".yaml")
|
name := strings.TrimSuffix(f.Name(), ".yaml")
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
name = strings.TrimSuffix(name, ".yml")
|
||||||
|
|
||||||
log.Debug().Msgf("Checking file %s", f.Name())
|
xlog.Debug("Checking file", "file", f.Name())
|
||||||
files, err := listModelFiles(systemState, name)
|
files, err := listModelFiles(systemState, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Msgf("failed to list files for model %s", f.Name())
|
xlog.Debug("failed to list files for model", "error", err, "model", f.Name())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
allOtherFiles = append(allOtherFiles, files...)
|
allOtherFiles = append(allOtherFiles, files...)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Files to remove: %+v", filesToRemove)
|
xlog.Debug("Files to remove", "files", filesToRemove)
|
||||||
log.Debug().Msgf("All other files: %+v", allOtherFiles)
|
xlog.Debug("All other files", "files", allOtherFiles)
|
||||||
|
|
||||||
// Removing files
|
// Removing files
|
||||||
for _, f := range filesToRemove {
|
for _, f := range filesToRemove {
|
||||||
if slices.Contains(allOtherFiles, f) {
|
if slices.Contains(allOtherFiles, f) {
|
||||||
log.Debug().Msgf("Skipping file %s because it is part of another model", f)
|
xlog.Debug("Skipping file because it is part of another model", "file", f)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if e := os.Remove(f); e != nil {
|
if e := os.Remove(f); e != nil {
|
||||||
log.Error().Err(e).Msgf("failed to remove file %s", f)
|
xlog.Error("failed to remove file", "error", e, "file", f)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,7 +436,7 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
|
|||||||
for _, file := range galleryModel.AdditionalFiles {
|
for _, file := range galleryModel.AdditionalFiles {
|
||||||
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
|
||||||
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
|
||||||
log.Error().Str("model", galleryModel.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
|
xlog.Error("Contains unsafe file(s)!", "model", galleryModel.Name, "clamAV", scanResults.ClamAVInfectedFiles, "pickles", scanResults.DangerousPickles)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Embed a directory
|
// Embed a directory
|
||||||
@@ -101,18 +101,13 @@ func API(application *application.Application) (*echo.Echo, error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Custom logger middleware using zerolog
|
// Custom logger middleware using xlog
|
||||||
e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
|
e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||||
return func(c echo.Context) error {
|
return func(c echo.Context) error {
|
||||||
req := c.Request()
|
req := c.Request()
|
||||||
res := c.Response()
|
res := c.Response()
|
||||||
start := log.Logger.Info()
|
|
||||||
err := next(c)
|
err := next(c)
|
||||||
start.
|
xlog.Info("HTTP request", "method", req.Method, "path", req.URL.Path, "status", res.Status)
|
||||||
Str("method", req.Method).
|
|
||||||
Str("path", req.URL.Path).
|
|
||||||
Int("status", res.Status).
|
|
||||||
Msg("HTTP request")
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -193,7 +188,7 @@ func API(application *application.Application) (*echo.Echo, error) {
|
|||||||
|
|
||||||
// CSRF middleware
|
// CSRF middleware
|
||||||
if application.ApplicationConfig().CSRF {
|
if application.ApplicationConfig().CSRF {
|
||||||
log.Debug().Msg("Enabling CSRF middleware. Tokens are now required for state-modifying requests")
|
xlog.Debug("Enabling CSRF middleware. Tokens are now required for state-modifying requests")
|
||||||
e.Use(middleware.CSRF())
|
e.Use(middleware.CSRF())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,7 +214,7 @@ func API(application *application.Application) (*echo.Echo, error) {
|
|||||||
|
|
||||||
// Log startup message
|
// Log startup message
|
||||||
e.Server.RegisterOnShutdown(func() {
|
e.Server.RegisterOnShutdown(func() {
|
||||||
log.Info().Msg("LocalAI API server shutting down")
|
xlog.Info("LocalAI API server shutting down")
|
||||||
})
|
})
|
||||||
|
|
||||||
return e, nil
|
return e, nil
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ import (
|
|||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
|
"github.com/mudler/xlog"
|
||||||
openaigo "github.com/otiai10/openaigo"
|
openaigo "github.com/otiai10/openaigo"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
"github.com/sashabaranov/go-openai"
|
"github.com/sashabaranov/go-openai"
|
||||||
"github.com/sashabaranov/go-openai/jsonschema"
|
"github.com/sashabaranov/go-openai/jsonschema"
|
||||||
)
|
)
|
||||||
@@ -378,7 +378,7 @@ var _ = Describe("API test", func() {
|
|||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
||||||
log.Error().Err(err).Msg("server error")
|
xlog.Error("server error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -710,7 +710,7 @@ parameters:
|
|||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
||||||
log.Error().Err(err).Msg("server error")
|
xlog.Error("server error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -924,7 +924,7 @@ parameters:
|
|||||||
Expect(err).ToNot(HaveOccurred())
|
Expect(err).ToNot(HaveOccurred())
|
||||||
go func() {
|
go func() {
|
||||||
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
||||||
log.Error().Err(err).Msg("server error")
|
xlog.Error("server error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -1405,7 +1405,7 @@ parameters:
|
|||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
if err := app.Start("127.0.0.1:9090"); err != nil && err != http.ErrServerClosed {
|
||||||
log.Error().Err(err).Msg("server error")
|
xlog.Error("server error", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SoundGenerationEndpoint is the ElevenLabs SoundGeneration endpoint https://elevenlabs.io/docs/api-reference/sound-generation
|
// SoundGenerationEndpoint is the ElevenLabs SoundGeneration endpoint https://elevenlabs.io/docs/api-reference/sound-generation
|
||||||
@@ -30,7 +30,7 @@ func SoundGenerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Sound Generation Request about to be sent to backend")
|
xlog.Debug("Sound Generation Request about to be sent to backend", "modelFile", "modelFile", "backend", cfg.Backend)
|
||||||
|
|
||||||
// TODO: Support uploading files?
|
// TODO: Support uploading files?
|
||||||
filePath, _, err := backend.SoundGeneration(input.Text, input.Duration, input.Temperature, input.DoSample, nil, nil, ml, appConfig, *cfg)
|
filePath, _, err := backend.SoundGeneration(input.Text, input.Duration, input.Temperature, input.DoSample, nil, nil, ml, appConfig, *cfg)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
|
// TTSEndpoint is the OpenAI Speech API endpoint https://platform.openai.com/docs/api-reference/audio/createSpeech
|
||||||
@@ -33,7 +33,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("modelName", input.ModelID).Msg("elevenlabs TTS request received")
|
xlog.Debug("elevenlabs TTS request received", "modelName", input.ModelID)
|
||||||
|
|
||||||
filePath, _, err := backend.ModelTTS(input.Text, voiceID, input.LanguageCode, ml, appConfig, *cfg)
|
filePath, _, err := backend.ModelTTS(input.Text, voiceID, input.LanguageCode, ml, appConfig, *cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// JINARerankEndpoint acts like the Jina reranker endpoint (https://jina.ai/reranker/)
|
// JINARerankEndpoint acts like the Jina reranker endpoint (https://jina.ai/reranker/)
|
||||||
@@ -31,7 +31,7 @@ func JINARerankEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("model", input.Model).Msg("JINA Rerank Request received")
|
xlog.Debug("JINA Rerank Request received", "model", input.Model)
|
||||||
var requestTopN int32
|
var requestTopN int32
|
||||||
docs := int32(len(input.Documents))
|
docs := int32(len(input.Documents))
|
||||||
if input.TopN == nil { // omit top_n to get all
|
if input.TopN == nil { // omit top_n to get all
|
||||||
|
|||||||
@@ -4,15 +4,15 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/labstack/echo/v4"
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/gallery"
|
"github.com/mudler/LocalAI/core/gallery"
|
||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BackendEndpointService struct {
|
type BackendEndpointService struct {
|
||||||
@@ -131,7 +131,7 @@ func (mgs *BackendEndpointService) ListBackendsEndpoint(systemState *system.Syst
|
|||||||
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
|
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
|
||||||
func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc {
|
func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc {
|
||||||
return func(c echo.Context) error {
|
return func(c echo.Context) error {
|
||||||
log.Debug().Msgf("Listing backend galleries %+v", mgs.galleries)
|
xlog.Debug("Listing backend galleries", "galleries", mgs.galleries)
|
||||||
dat, err := json.Marshal(mgs.galleries)
|
dat, err := json.Marshal(mgs.galleries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DetectionEndpoint is the LocalAI Detection endpoint https://localai.io/docs/api-reference/detection
|
// DetectionEndpoint is the LocalAI Detection endpoint https://localai.io/docs/api-reference/detection
|
||||||
@@ -29,7 +29,7 @@ func DetectionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appC
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("image", input.Image).Str("modelFile", "modelFile").Str("backend", cfg.Backend).Msg("Detection")
|
xlog.Debug("Detection", "image", input.Image, "modelFile", "modelFile", "backend", cfg.Backend)
|
||||||
|
|
||||||
image, err := utils.GetContentURIAsBase64(input.Image)
|
image, err := utils.GetContentURIAsBase64(input.Image)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,15 +4,15 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/labstack/echo/v4"
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/gallery"
|
"github.com/mudler/LocalAI/core/gallery"
|
||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ModelGalleryEndpointService struct {
|
type ModelGalleryEndpointService struct {
|
||||||
@@ -121,11 +121,11 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
|
|||||||
|
|
||||||
models, err := gallery.AvailableGalleryModels(mgs.galleries, systemState)
|
models, err := gallery.AvailableGalleryModels(mgs.galleries, systemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("could not list models from galleries")
|
xlog.Error("could not list models from galleries", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Available %d models from %d galleries\n", len(models), len(mgs.galleries))
|
xlog.Debug("Available models from galleries", "modelCount", len(models), "galleryCount", len(mgs.galleries))
|
||||||
|
|
||||||
m := []gallery.Metadata{}
|
m := []gallery.Metadata{}
|
||||||
|
|
||||||
@@ -133,7 +133,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
|
|||||||
m = append(m, mm.Metadata)
|
m = append(m, mm.Metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Models %#v", m)
|
xlog.Debug("Models", "models", m)
|
||||||
|
|
||||||
dat, err := json.Marshal(m)
|
dat, err := json.Marshal(m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -150,7 +150,7 @@ func (mgs *ModelGalleryEndpointService) ListModelFromGalleryEndpoint(systemState
|
|||||||
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
|
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
|
||||||
func (mgs *ModelGalleryEndpointService) ListModelGalleriesEndpoint() echo.HandlerFunc {
|
func (mgs *ModelGalleryEndpointService) ListModelGalleriesEndpoint() echo.HandlerFunc {
|
||||||
return func(c echo.Context) error {
|
return func(c echo.Context) error {
|
||||||
log.Debug().Msgf("Listing model galleries %+v", mgs.galleries)
|
xlog.Debug("Listing model galleries", "galleries", mgs.galleries)
|
||||||
dat, err := json.Marshal(mgs.galleries)
|
dat, err := json.Marshal(mgs.galleries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
)
|
)
|
||||||
@@ -34,19 +34,19 @@ func TokenMetricsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a
|
|||||||
modelFile, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
modelFile, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
||||||
if !ok || modelFile != "" {
|
if !ok || modelFile != "" {
|
||||||
modelFile = input.Model
|
modelFile = input.Model
|
||||||
log.Warn().Msgf("Model not found in context: %s", input.Model)
|
xlog.Warn("Model not found in context", "model", input.Model)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(modelFile, appConfig)
|
cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(modelFile, appConfig)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Err(err)
|
xlog.Error("Error loading model config", "error", err)
|
||||||
modelFile = input.Model
|
modelFile = input.Model
|
||||||
log.Warn().Msgf("Model not found in context: %s", input.Model)
|
xlog.Warn("Model not found in context", "model", input.Model)
|
||||||
} else {
|
} else {
|
||||||
modelFile = cfg.Model
|
modelFile = cfg.Model
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Token Metrics for model: %s", modelFile)
|
xlog.Debug("Token Metrics for model", "model", modelFile)
|
||||||
|
|
||||||
response, err := backend.TokenMetrics(modelFile, ml, appConfig, *cfg)
|
response, err := backend.TokenMetrics(modelFile, ml, appConfig, *cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/templates"
|
"github.com/mudler/LocalAI/core/templates"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/cogito"
|
"github.com/mudler/cogito"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MCP SSE Event Types
|
// MCP SSE Event Types
|
||||||
@@ -138,19 +138,19 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
cogitoOpts = append(
|
cogitoOpts = append(
|
||||||
cogitoOpts,
|
cogitoOpts,
|
||||||
cogito.WithStatusCallback(func(s string) {
|
cogito.WithStatusCallback(func(s string) {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s)
|
xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
|
||||||
}),
|
}),
|
||||||
cogito.WithReasoningCallback(func(s string) {
|
cogito.WithReasoningCallback(func(s string) {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s)
|
xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
||||||
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).Msg("[model agent] Tool call")
|
xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
|
||||||
return cogito.ToolCallDecision{
|
return cogito.ToolCallDecision{
|
||||||
Approved: true,
|
Approved: true,
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
||||||
log.Debug().Str("model", config.Name).Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).Msg("[model agent] Tool call result")
|
xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -176,7 +176,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
@@ -279,7 +279,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
// Context was cancelled (client disconnected or request cancelled)
|
// Context was cancelled (client disconnected or request cancelled)
|
||||||
log.Debug().Msgf("Request context cancelled, stopping stream")
|
xlog.Debug("Request context cancelled, stopping stream")
|
||||||
cancel()
|
cancel()
|
||||||
break LOOP
|
break LOOP
|
||||||
case event := <-events:
|
case event := <-events:
|
||||||
@@ -289,13 +289,13 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
}
|
}
|
||||||
eventData, err := json.Marshal(event)
|
eventData, err := json.Marshal(event)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Failed to marshal event: %v", err)
|
xlog.Debug("Failed to marshal event", "error", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Sending event: %s", string(eventData))
|
xlog.Debug("Sending event", "event", string(eventData))
|
||||||
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(eventData))
|
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(eventData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Sending event failed: %v", err)
|
xlog.Debug("Sending event failed", "error", err)
|
||||||
cancel()
|
cancel()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -307,7 +307,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
c.Response().Flush()
|
c.Response().Flush()
|
||||||
break LOOP
|
break LOOP
|
||||||
}
|
}
|
||||||
log.Error().Msgf("Stream ended with error: %v", err)
|
xlog.Error("Stream ended with error", "error", err)
|
||||||
errorEvent := MCPErrorEvent{
|
errorEvent := MCPErrorEvent{
|
||||||
Type: "error",
|
Type: "error",
|
||||||
Message: err.Error(),
|
Message: err.Error(),
|
||||||
@@ -324,7 +324,7 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Stream ended")
|
xlog.Debug("Stream ended")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/p2p"
|
"github.com/mudler/LocalAI/core/p2p"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetSettingsEndpoint returns current settings with precedence (env > file > defaults)
|
// GetSettingsEndpoint returns current settings with precedence (env > file > defaults)
|
||||||
@@ -118,7 +118,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
|
|||||||
if watchdogChanged {
|
if watchdogChanged {
|
||||||
if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled {
|
if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled {
|
||||||
if err := app.StopWatchdog(); err != nil {
|
if err := app.StopWatchdog(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to stop watchdog")
|
xlog.Error("Failed to stop watchdog", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
||||||
Success: false,
|
Success: false,
|
||||||
Error: "Settings saved but failed to stop watchdog: " + err.Error(),
|
Error: "Settings saved but failed to stop watchdog: " + err.Error(),
|
||||||
@@ -126,7 +126,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err := app.RestartWatchdog(); err != nil {
|
if err := app.RestartWatchdog(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to restart watchdog")
|
xlog.Error("Failed to restart watchdog", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
||||||
Success: false,
|
Success: false,
|
||||||
Error: "Settings saved but failed to restart watchdog: " + err.Error(),
|
Error: "Settings saved but failed to restart watchdog: " + err.Error(),
|
||||||
@@ -138,7 +138,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
|
|||||||
// Restart agent job service if retention days changed
|
// Restart agent job service if retention days changed
|
||||||
if agentJobChanged {
|
if agentJobChanged {
|
||||||
if err := app.RestartAgentJobService(); err != nil {
|
if err := app.RestartAgentJobService(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to restart agent job service")
|
xlog.Error("Failed to restart agent job service", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
||||||
Success: false,
|
Success: false,
|
||||||
Error: "Settings saved but failed to restart agent job service: " + err.Error(),
|
Error: "Settings saved but failed to restart agent job service: " + err.Error(),
|
||||||
@@ -151,7 +151,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
|
|||||||
if p2pChanged {
|
if p2pChanged {
|
||||||
if settings.P2PToken != nil && *settings.P2PToken == "" {
|
if settings.P2PToken != nil && *settings.P2PToken == "" {
|
||||||
if err := app.StopP2P(); err != nil {
|
if err := app.StopP2P(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to stop P2P")
|
xlog.Error("Failed to stop P2P", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
||||||
Success: false,
|
Success: false,
|
||||||
Error: "Settings saved but failed to stop P2P: " + err.Error(),
|
Error: "Settings saved but failed to stop P2P: " + err.Error(),
|
||||||
@@ -164,7 +164,7 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
|
|||||||
appConfig.P2PToken = token
|
appConfig.P2PToken = token
|
||||||
}
|
}
|
||||||
if err := app.RestartP2P(); err != nil {
|
if err := app.RestartP2P(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to restart P2P")
|
xlog.Error("Failed to restart P2P", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
return c.JSON(http.StatusInternalServerError, schema.SettingsResponse{
|
||||||
Success: false,
|
Success: false,
|
||||||
Error: "Settings saved but failed to restart P2P: " + err.Error(),
|
Error: "Settings saved but failed to restart P2P: " + err.Error(),
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
)
|
)
|
||||||
@@ -36,7 +36,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("model", input.Model).Msg("LocalAI TTS Request received")
|
xlog.Debug("LocalAI TTS Request received", "model", input.Model)
|
||||||
|
|
||||||
if cfg.Backend == "" && input.Backend != "" {
|
if cfg.Backend == "" && input.Backend != "" {
|
||||||
cfg.Backend = input.Backend
|
cfg.Backend = input.Backend
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// VADEndpoint is Voice-Activation-Detection endpoint
|
// VADEndpoint is Voice-Activation-Detection endpoint
|
||||||
@@ -28,7 +28,7 @@ func VADEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("model", input.Model).Msg("LocalAI VAD Request received")
|
xlog.Debug("LocalAI VAD Request received", "model", input.Model)
|
||||||
|
|
||||||
resp, err := backend.VAD(input, c.Request().Context(), ml, appConfig, *cfg)
|
resp, err := backend.VAD(input, c.Request().Context(), ml, appConfig, *cfg)
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/backend"
|
"github.com/mudler/LocalAI/core/backend"
|
||||||
|
|
||||||
model "github.com/mudler/LocalAI/pkg/model"
|
model "github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func downloadFile(url string) (string, error) {
|
func downloadFile(url string) (string, error) {
|
||||||
@@ -69,13 +69,13 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
return func(c echo.Context) error {
|
return func(c echo.Context) error {
|
||||||
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.VideoRequest)
|
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.VideoRequest)
|
||||||
if !ok || input.Model == "" {
|
if !ok || input.Model == "" {
|
||||||
log.Error().Msg("Video Endpoint - Invalid Input")
|
xlog.Error("Video Endpoint - Invalid Input")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
||||||
if !ok || config == nil {
|
if !ok || config == nil {
|
||||||
log.Error().Msg("Video Endpoint - Invalid Config")
|
xlog.Error("Video Endpoint - Invalid Config")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,7 +124,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
defer os.RemoveAll(src)
|
defer os.RemoveAll(src)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Parameter Config: %+v", config)
|
xlog.Debug("Parameter Config", "config", config)
|
||||||
|
|
||||||
switch config.Backend {
|
switch config.Backend {
|
||||||
case "stablediffusion":
|
case "stablediffusion":
|
||||||
@@ -217,7 +217,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/signals"
|
"github.com/mudler/LocalAI/pkg/signals"
|
||||||
|
|
||||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type sessionCache struct {
|
type sessionCache struct {
|
||||||
@@ -47,7 +47,7 @@ func SessionsFromMCPConfig(
|
|||||||
|
|
||||||
// Get the list of all the tools that the Agent will be esposed to
|
// Get the list of all the tools that the Agent will be esposed to
|
||||||
for _, server := range remote.Servers {
|
for _, server := range remote.Servers {
|
||||||
log.Debug().Msgf("[MCP remote server] Configuration : %+v", server)
|
xlog.Debug("[MCP remote server] Configuration", "server", server)
|
||||||
// Create HTTP client with custom roundtripper for bearer token injection
|
// Create HTTP client with custom roundtripper for bearer token injection
|
||||||
httpClient := &http.Client{
|
httpClient := &http.Client{
|
||||||
Timeout: 360 * time.Second,
|
Timeout: 360 * time.Second,
|
||||||
@@ -57,16 +57,16 @@ func SessionsFromMCPConfig(
|
|||||||
transport := &mcp.StreamableClientTransport{Endpoint: server.URL, HTTPClient: httpClient}
|
transport := &mcp.StreamableClientTransport{Endpoint: server.URL, HTTPClient: httpClient}
|
||||||
mcpSession, err := client.Connect(ctx, transport, nil)
|
mcpSession, err := client.Connect(ctx, transport, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed to connect to MCP server %s", server.URL)
|
xlog.Error("Failed to connect to MCP server", "error", err, "url", server.URL)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("[MCP remote server] Connected to MCP server %s", server.URL)
|
xlog.Debug("[MCP remote server] Connected to MCP server", "url", server.URL)
|
||||||
cache.cache[name] = append(cache.cache[name], mcpSession)
|
cache.cache[name] = append(cache.cache[name], mcpSession)
|
||||||
allSessions = append(allSessions, mcpSession)
|
allSessions = append(allSessions, mcpSession)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, server := range stdio.Servers {
|
for _, server := range stdio.Servers {
|
||||||
log.Debug().Msgf("[MCP stdio server] Configuration : %+v", server)
|
xlog.Debug("[MCP stdio server] Configuration", "server", server)
|
||||||
command := exec.Command(server.Command, server.Args...)
|
command := exec.Command(server.Command, server.Args...)
|
||||||
command.Env = os.Environ()
|
command.Env = os.Environ()
|
||||||
for key, value := range server.Env {
|
for key, value := range server.Env {
|
||||||
@@ -75,10 +75,10 @@ func SessionsFromMCPConfig(
|
|||||||
transport := &mcp.CommandTransport{Command: command}
|
transport := &mcp.CommandTransport{Command: command}
|
||||||
mcpSession, err := client.Connect(ctx, transport, nil)
|
mcpSession, err := client.Connect(ctx, transport, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed to start MCP server %s", command)
|
xlog.Error("Failed to start MCP server", "error", err, "command", command)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("[MCP stdio server] Connected to MCP server %s", command)
|
xlog.Debug("[MCP stdio server] Connected to MCP server", "command", command)
|
||||||
cache.cache[name] = append(cache.cache[name], mcpSession)
|
cache.cache[name] = append(cache.cache[name], mcpSession)
|
||||||
allSessions = append(allSessions, mcpSession)
|
allSessions = append(allSessions, mcpSession)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/templates"
|
"github.com/mudler/LocalAI/core/templates"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create
|
// ChatEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/chat/create
|
||||||
@@ -78,7 +78,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
textContentToReturn = functions.ParseTextContent(result, config.FunctionsConfig)
|
textContentToReturn = functions.ParseTextContent(result, config.FunctionsConfig)
|
||||||
result = functions.CleanupLLMResult(result, config.FunctionsConfig)
|
result = functions.CleanupLLMResult(result, config.FunctionsConfig)
|
||||||
functionResults := functions.ParseFunctionCall(result, config.FunctionsConfig)
|
functionResults := functions.ParseFunctionCall(result, config.FunctionsConfig)
|
||||||
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
|
xlog.Debug("Text content to return", "text", textContentToReturn)
|
||||||
noActionToRun := len(functionResults) > 0 && functionResults[0].Name == noAction || len(functionResults) == 0
|
noActionToRun := len(functionResults) > 0 && functionResults[0].Name == noAction || len(functionResults) == 0
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
@@ -94,7 +94,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
|
|
||||||
result, err := handleQuestion(config, cl, req, ml, startupOptions, functionResults, result, prompt)
|
result, err := handleQuestion(config, cl, req, ml, startupOptions, functionResults, result, prompt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error handling question")
|
xlog.Error("error handling question", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
usage := schema.OpenAIUsage{
|
usage := schema.OpenAIUsage{
|
||||||
@@ -195,7 +195,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Chat endpoint configuration read: %+v", config)
|
xlog.Debug("Chat endpoint configuration read", "config", config)
|
||||||
|
|
||||||
funcs := input.Functions
|
funcs := input.Functions
|
||||||
shouldUseFn := len(input.Functions) > 0 && config.ShouldUseFunctions()
|
shouldUseFn := len(input.Functions) > 0 && config.ShouldUseFunctions()
|
||||||
@@ -252,7 +252,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
input.Grammar = g
|
input.Grammar = g
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Msg("Failed generating grammar")
|
xlog.Error("Failed generating grammar", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -260,7 +260,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
config.Grammar = input.Grammar
|
config.Grammar = input.Grammar
|
||||||
|
|
||||||
if shouldUseFn {
|
if shouldUseFn {
|
||||||
log.Debug().Msgf("Response needs to process functions")
|
xlog.Debug("Response needs to process functions")
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
@@ -294,14 +294,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
config.Grammar = g
|
config.Grammar = g
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Msg("Failed generating grammar")
|
xlog.Error("Failed generating grammar", "error", err)
|
||||||
}
|
}
|
||||||
case input.JSONFunctionGrammarObject != nil:
|
case input.JSONFunctionGrammarObject != nil:
|
||||||
g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarOptions()...)
|
g, err := input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarOptions()...)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
config.Grammar = g
|
config.Grammar = g
|
||||||
} else {
|
} else {
|
||||||
log.Error().Err(err).Msg("Failed generating grammar")
|
xlog.Error("Failed generating grammar", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -316,7 +316,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
// functions are not supported in stream mode (yet?)
|
// functions are not supported in stream mode (yet?)
|
||||||
toStream := input.Stream
|
toStream := input.Stream
|
||||||
|
|
||||||
log.Debug().Msgf("Parameters: %+v", config)
|
xlog.Debug("Parameters", "config", config)
|
||||||
|
|
||||||
var predInput string
|
var predInput string
|
||||||
|
|
||||||
@@ -325,16 +325,16 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
if !config.TemplateConfig.UseTokenizerTemplate {
|
if !config.TemplateConfig.UseTokenizerTemplate {
|
||||||
predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
|
predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
|
||||||
|
|
||||||
log.Debug().Msgf("Prompt (after templating): %s", predInput)
|
xlog.Debug("Prompt (after templating)", "prompt", predInput)
|
||||||
if config.Grammar != "" {
|
if config.Grammar != "" {
|
||||||
log.Debug().Msgf("Grammar: %+v", config.Grammar)
|
xlog.Debug("Grammar", "grammar", config.Grammar)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case toStream:
|
case toStream:
|
||||||
|
|
||||||
log.Debug().Msgf("Stream request received")
|
xlog.Debug("Stream request received")
|
||||||
c.Response().Header().Set("Content-Type", "text/event-stream")
|
c.Response().Header().Set("Content-Type", "text/event-stream")
|
||||||
c.Response().Header().Set("Cache-Control", "no-cache")
|
c.Response().Header().Set("Cache-Control", "no-cache")
|
||||||
c.Response().Header().Set("Connection", "keep-alive")
|
c.Response().Header().Set("Connection", "keep-alive")
|
||||||
@@ -359,12 +359,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
select {
|
select {
|
||||||
case <-input.Context.Done():
|
case <-input.Context.Done():
|
||||||
// Context was cancelled (client disconnected or request cancelled)
|
// Context was cancelled (client disconnected or request cancelled)
|
||||||
log.Debug().Msgf("Request context cancelled, stopping stream")
|
xlog.Debug("Request context cancelled, stopping stream")
|
||||||
input.Cancel()
|
input.Cancel()
|
||||||
break LOOP
|
break LOOP
|
||||||
case ev := <-responses:
|
case ev := <-responses:
|
||||||
if len(ev.Choices) == 0 {
|
if len(ev.Choices) == 0 {
|
||||||
log.Debug().Msgf("No choices in the response, skipping")
|
xlog.Debug("No choices in the response, skipping")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
usage = &ev.Usage // Copy a pointer to the latest usage chunk so that the stop message can reference it
|
usage = &ev.Usage // Copy a pointer to the latest usage chunk so that the stop message can reference it
|
||||||
@@ -373,14 +373,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
}
|
}
|
||||||
respData, err := json.Marshal(ev)
|
respData, err := json.Marshal(ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Failed to marshal response: %v", err)
|
xlog.Debug("Failed to marshal response", "error", err)
|
||||||
input.Cancel()
|
input.Cancel()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Sending chunk: %s", string(respData))
|
xlog.Debug("Sending chunk", "chunk", string(respData))
|
||||||
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
|
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Sending chunk failed: %v", err)
|
xlog.Debug("Sending chunk failed", "error", err)
|
||||||
input.Cancel()
|
input.Cancel()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -389,7 +389,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
break LOOP
|
break LOOP
|
||||||
}
|
}
|
||||||
log.Error().Msgf("Stream ended with error: %v", err)
|
xlog.Error("Stream ended with error", "error", err)
|
||||||
|
|
||||||
stopReason := FinishReasonStop
|
stopReason := FinishReasonStop
|
||||||
resp := &schema.OpenAIResponse{
|
resp := &schema.OpenAIResponse{
|
||||||
@@ -407,7 +407,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
}
|
}
|
||||||
respData, marshalErr := json.Marshal(resp)
|
respData, marshalErr := json.Marshal(resp)
|
||||||
if marshalErr != nil {
|
if marshalErr != nil {
|
||||||
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
|
xlog.Error("Failed to marshal error response", "error", marshalErr)
|
||||||
// Send a simple error message as fallback
|
// Send a simple error message as fallback
|
||||||
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
|
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
|
||||||
} else {
|
} else {
|
||||||
@@ -445,7 +445,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData)
|
fmt.Fprintf(c.Response().Writer, "data: %s\n\n", respData)
|
||||||
fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n")
|
fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n")
|
||||||
c.Response().Flush()
|
c.Response().Flush()
|
||||||
log.Debug().Msgf("Stream ended")
|
xlog.Debug("Stream ended")
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
// no streaming mode
|
// no streaming mode
|
||||||
@@ -462,14 +462,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
|
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
|
||||||
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
|
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
|
||||||
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
|
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
|
||||||
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
|
xlog.Debug("Text content to return", "text", textContentToReturn)
|
||||||
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
|
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case noActionsToRun:
|
case noActionsToRun:
|
||||||
result, err := handleQuestion(config, cl, input, ml, startupOptions, results, s, predInput)
|
result, err := handleQuestion(config, cl, input, ml, startupOptions, results, s, predInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error handling question")
|
xlog.Error("error handling question", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -562,7 +562,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
Usage: usage,
|
Usage: usage,
|
||||||
}
|
}
|
||||||
respData, _ := json.Marshal(resp)
|
respData, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", respData)
|
xlog.Debug("Response", "response", string(respData))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
@@ -573,12 +573,12 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, input *schema.OpenAIRequest, ml *model.ModelLoader, o *config.ApplicationConfig, funcResults []functions.FuncCallResults, result, prompt string) (string, error) {
|
func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, input *schema.OpenAIRequest, ml *model.ModelLoader, o *config.ApplicationConfig, funcResults []functions.FuncCallResults, result, prompt string) (string, error) {
|
||||||
|
|
||||||
if len(funcResults) == 0 && result != "" {
|
if len(funcResults) == 0 && result != "" {
|
||||||
log.Debug().Msgf("nothing function results but we had a message from the LLM")
|
xlog.Debug("nothing function results but we had a message from the LLM")
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("nothing to do, computing a reply")
|
xlog.Debug("nothing to do, computing a reply")
|
||||||
arg := ""
|
arg := ""
|
||||||
if len(funcResults) > 0 {
|
if len(funcResults) > 0 {
|
||||||
arg = funcResults[0].Arguments
|
arg = funcResults[0].Arguments
|
||||||
@@ -586,23 +586,23 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
|
|||||||
// If there is a message that the LLM already sends as part of the JSON reply, use it
|
// If there is a message that the LLM already sends as part of the JSON reply, use it
|
||||||
arguments := map[string]interface{}{}
|
arguments := map[string]interface{}{}
|
||||||
if err := json.Unmarshal([]byte(arg), &arguments); err != nil {
|
if err := json.Unmarshal([]byte(arg), &arguments); err != nil {
|
||||||
log.Debug().Msg("handleQuestion: function result did not contain a valid JSON object")
|
xlog.Debug("handleQuestion: function result did not contain a valid JSON object")
|
||||||
}
|
}
|
||||||
m, exists := arguments["message"]
|
m, exists := arguments["message"]
|
||||||
if exists {
|
if exists {
|
||||||
switch message := m.(type) {
|
switch message := m.(type) {
|
||||||
case string:
|
case string:
|
||||||
if message != "" {
|
if message != "" {
|
||||||
log.Debug().Msgf("Reply received from LLM: %s", message)
|
xlog.Debug("Reply received from LLM", "message", message)
|
||||||
message = backend.Finetune(*config, prompt, message)
|
message = backend.Finetune(*config, prompt, message)
|
||||||
log.Debug().Msgf("Reply received from LLM(finetuned): %s", message)
|
xlog.Debug("Reply received from LLM(finetuned)", "message", message)
|
||||||
|
|
||||||
return message, nil
|
return message, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("No action received from LLM, without a message, computing a reply")
|
xlog.Debug("No action received from LLM, without a message, computing a reply")
|
||||||
// Otherwise ask the LLM to understand the JSON output and the context, and return a message
|
// Otherwise ask the LLM to understand the JSON output and the context, and return a message
|
||||||
// Note: This costs (in term of CPU/GPU) another computation
|
// Note: This costs (in term of CPU/GPU) another computation
|
||||||
config.Grammar = ""
|
config.Grammar = ""
|
||||||
@@ -662,13 +662,13 @@ func handleQuestion(config *config.ModelConfig, cl *config.ModelConfigLoader, in
|
|||||||
|
|
||||||
predFunc, err := backend.ModelInference(input.Context, prompt, input.Messages, images, videos, audios, ml, config, cl, o, nil, toolsJSON, toolChoiceJSON, logprobs, topLogprobs, logitBias)
|
predFunc, err := backend.ModelInference(input.Context, prompt, input.Messages, images, videos, audios, ml, config, cl, o, nil, toolsJSON, toolChoiceJSON, logprobs, topLogprobs, logitBias)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("model inference failed")
|
xlog.Error("model inference failed", "error", err)
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
prediction, err := predFunc()
|
prediction, err := predFunc()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("prediction failed")
|
xlog.Error("prediction failed", "error", err)
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return backend.Finetune(*config, prompt, prediction.Response), nil
|
return backend.Finetune(*config, prompt, prediction.Response), nil
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/templates"
|
"github.com/mudler/LocalAI/core/templates"
|
||||||
"github.com/mudler/LocalAI/pkg/functions"
|
"github.com/mudler/LocalAI/pkg/functions"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
|
// CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
|
||||||
@@ -52,7 +52,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
Object: "text_completion",
|
Object: "text_completion",
|
||||||
Usage: usage,
|
Usage: usage,
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Sending goroutine: %s", s)
|
xlog.Debug("Sending goroutine", "text", s)
|
||||||
|
|
||||||
responses <- resp
|
responses <- resp
|
||||||
return true
|
return true
|
||||||
@@ -94,10 +94,10 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
|
|
||||||
config.Grammar = input.Grammar
|
config.Grammar = input.Grammar
|
||||||
|
|
||||||
log.Debug().Msgf("Parameter Config: %+v", config)
|
xlog.Debug("Parameter Config", "config", config)
|
||||||
|
|
||||||
if input.Stream {
|
if input.Stream {
|
||||||
log.Debug().Msgf("Stream request received")
|
xlog.Debug("Stream request received")
|
||||||
c.Response().Header().Set("Content-Type", "text/event-stream")
|
c.Response().Header().Set("Content-Type", "text/event-stream")
|
||||||
c.Response().Header().Set("Cache-Control", "no-cache")
|
c.Response().Header().Set("Cache-Control", "no-cache")
|
||||||
c.Response().Header().Set("Connection", "keep-alive")
|
c.Response().Header().Set("Connection", "keep-alive")
|
||||||
@@ -116,7 +116,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
predInput = templatedInput
|
predInput = templatedInput
|
||||||
log.Debug().Msgf("Template found, input modified to: %s", predInput)
|
xlog.Debug("Template found, input modified", "input", predInput)
|
||||||
}
|
}
|
||||||
|
|
||||||
responses := make(chan schema.OpenAIResponse)
|
responses := make(chan schema.OpenAIResponse)
|
||||||
@@ -131,16 +131,16 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
select {
|
select {
|
||||||
case ev := <-responses:
|
case ev := <-responses:
|
||||||
if len(ev.Choices) == 0 {
|
if len(ev.Choices) == 0 {
|
||||||
log.Debug().Msgf("No choices in the response, skipping")
|
xlog.Debug("No choices in the response, skipping")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
respData, err := json.Marshal(ev)
|
respData, err := json.Marshal(ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Failed to marshal response: %v", err)
|
xlog.Debug("Failed to marshal response", "error", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Sending chunk: %s", string(respData))
|
xlog.Debug("Sending chunk", "chunk", string(respData))
|
||||||
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
|
_, err = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", string(respData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -150,7 +150,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
break LOOP
|
break LOOP
|
||||||
}
|
}
|
||||||
log.Error().Msgf("Stream ended with error: %v", err)
|
xlog.Error("Stream ended with error", "error", err)
|
||||||
|
|
||||||
stopReason := FinishReasonStop
|
stopReason := FinishReasonStop
|
||||||
errorResp := schema.OpenAIResponse{
|
errorResp := schema.OpenAIResponse{
|
||||||
@@ -168,7 +168,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
}
|
}
|
||||||
errorData, marshalErr := json.Marshal(errorResp)
|
errorData, marshalErr := json.Marshal(errorResp)
|
||||||
if marshalErr != nil {
|
if marshalErr != nil {
|
||||||
log.Error().Msgf("Failed to marshal error response: %v", marshalErr)
|
xlog.Error("Failed to marshal error response", "error", marshalErr)
|
||||||
// Send a simple error message as fallback
|
// Send a simple error message as fallback
|
||||||
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
|
fmt.Fprintf(c.Response().Writer, "data: {\"error\":\"Internal error\"}\n\n")
|
||||||
} else {
|
} else {
|
||||||
@@ -213,7 +213,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
i = templatedInput
|
i = templatedInput
|
||||||
log.Debug().Msgf("Template found, input modified to: %s", i)
|
xlog.Debug("Template found, input modified", "input", i)
|
||||||
}
|
}
|
||||||
|
|
||||||
r, tokenUsage, err := ComputeChoices(
|
r, tokenUsage, err := ComputeChoices(
|
||||||
@@ -250,7 +250,7 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/templates"
|
"github.com/mudler/LocalAI/core/templates"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// EditEndpoint is the OpenAI edit API endpoint
|
// EditEndpoint is the OpenAI edit API endpoint
|
||||||
@@ -39,8 +39,8 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Edit Endpoint Input : %+v", input)
|
xlog.Debug("Edit Endpoint Input", "input", input)
|
||||||
log.Debug().Msgf("Edit Endpoint Config: %+v", *config)
|
xlog.Debug("Edit Endpoint Config", "config", *config)
|
||||||
|
|
||||||
var result []schema.Choice
|
var result []schema.Choice
|
||||||
totalTokenUsage := backend.TokenUsage{}
|
totalTokenUsage := backend.TokenUsage{}
|
||||||
@@ -55,7 +55,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
i = templatedInput
|
i = templatedInput
|
||||||
log.Debug().Msgf("Template found, input modified to: %s", i)
|
xlog.Debug("Template found, input modified", "input", i)
|
||||||
}
|
}
|
||||||
|
|
||||||
r, tokenUsage, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
|
r, tokenUsage, err := ComputeChoices(input, i, config, cl, appConfig, ml, func(s string, c *[]schema.Choice) {
|
||||||
@@ -95,7 +95,7 @@ func EditEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings
|
// EmbeddingsEndpoint is the OpenAI Embeddings API endpoint https://platform.openai.com/docs/api-reference/embeddings
|
||||||
@@ -33,7 +33,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Parameter Config: %+v", config)
|
xlog.Debug("Parameter Config", "config", config)
|
||||||
items := []schema.Item{}
|
items := []schema.Item{}
|
||||||
|
|
||||||
for i, s := range config.InputToken {
|
for i, s := range config.InputToken {
|
||||||
@@ -75,7 +75,7 @@ func EmbeddingsEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/backend"
|
"github.com/mudler/LocalAI/core/backend"
|
||||||
|
|
||||||
model "github.com/mudler/LocalAI/pkg/model"
|
model "github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func downloadFile(url string) (string, error) {
|
func downloadFile(url string) (string, error) {
|
||||||
@@ -70,13 +70,13 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
return func(c echo.Context) error {
|
return func(c echo.Context) error {
|
||||||
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
|
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
|
||||||
if !ok || input.Model == "" {
|
if !ok || input.Model == "" {
|
||||||
log.Error().Msg("Image Endpoint - Invalid Input")
|
xlog.Error("Image Endpoint - Invalid Input")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
||||||
if !ok || config == nil {
|
if !ok || config == nil {
|
||||||
log.Error().Msg("Image Endpoint - Invalid Config")
|
xlog.Error("Image Endpoint - Invalid Config")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +113,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Parameter Config: %+v", config)
|
xlog.Debug("Parameter Config", "config", config)
|
||||||
|
|
||||||
switch config.Backend {
|
switch config.Backend {
|
||||||
case "stablediffusion":
|
case "stablediffusion":
|
||||||
@@ -124,7 +124,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
|
|
||||||
if !strings.Contains(input.Size, "x") {
|
if !strings.Contains(input.Size, "x") {
|
||||||
input.Size = "512x512"
|
input.Size = "512x512"
|
||||||
log.Warn().Msgf("Invalid size, using default 512x512")
|
xlog.Warn("Invalid size, using default 512x512")
|
||||||
}
|
}
|
||||||
|
|
||||||
sizeParts := strings.Split(input.Size, "x")
|
sizeParts := strings.Split(input.Size, "x")
|
||||||
@@ -235,7 +235,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
@@ -251,21 +251,21 @@ func processImageFile(file string, generatedContentDir string) string {
|
|||||||
if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") {
|
if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") {
|
||||||
out, err := downloadFile(file)
|
out, err := downloadFile(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed downloading file: %s", file)
|
xlog.Error("Failed downloading file", "error", err, "file", file)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(out)
|
defer os.RemoveAll(out)
|
||||||
|
|
||||||
fileData, err = os.ReadFile(out)
|
fileData, err = os.ReadFile(out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed reading downloaded file: %s", out)
|
xlog.Error("Failed reading downloaded file", "error", err, "file", out)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// base 64 decode the file and write it somewhere that we will cleanup
|
// base 64 decode the file and write it somewhere that we will cleanup
|
||||||
fileData, err = base64.StdEncoding.DecodeString(file)
|
fileData, err = base64.StdEncoding.DecodeString(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed decoding base64 file")
|
xlog.Error("Failed decoding base64 file", "error", err)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -273,7 +273,7 @@ func processImageFile(file string, generatedContentDir string) string {
|
|||||||
// Create a temporary file
|
// Create a temporary file
|
||||||
outputFile, err := os.CreateTemp(generatedContentDir, "b64")
|
outputFile, err := os.CreateTemp(generatedContentDir, "b64")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Failed creating temporary file")
|
xlog.Error("Failed creating temporary file", "error", err)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -282,7 +282,7 @@ func processImageFile(file string, generatedContentDir string) string {
|
|||||||
_, err = writer.Write(fileData)
|
_, err = writer.Write(fileData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
outputFile.Close()
|
outputFile.Close()
|
||||||
log.Error().Err(err).Msg("Failed writing to temporary file")
|
xlog.Error("Failed writing to temporary file", "error", err)
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
outputFile.Close()
|
outputFile.Close()
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/labstack/echo/v4"
|
"github.com/labstack/echo/v4"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/backend"
|
"github.com/mudler/LocalAI/core/backend"
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
@@ -48,7 +48,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
stepsStr := c.FormValue("steps")
|
stepsStr := c.FormValue("steps")
|
||||||
|
|
||||||
if modelName == "" || prompt == "" {
|
if modelName == "" || prompt == "" {
|
||||||
log.Error().Msg("Inpainting Endpoint - missing model or prompt")
|
xlog.Error("Inpainting Endpoint - missing model or prompt")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,12 +63,12 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
// Get uploaded files
|
// Get uploaded files
|
||||||
imageFile, err := c.FormFile("image")
|
imageFile, err := c.FormFile("image")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Inpainting Endpoint - missing image file")
|
xlog.Error("Inpainting Endpoint - missing image file", "error", err)
|
||||||
return echo.NewHTTPError(http.StatusBadRequest, "missing image file")
|
return echo.NewHTTPError(http.StatusBadRequest, "missing image file")
|
||||||
}
|
}
|
||||||
maskFile, err := c.FormFile("mask")
|
maskFile, err := c.FormFile("mask")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Inpainting Endpoint - missing mask file")
|
xlog.Error("Inpainting Endpoint - missing mask file", "error", err)
|
||||||
return echo.NewHTTPError(http.StatusBadRequest, "missing mask file")
|
return echo.NewHTTPError(http.StatusBadRequest, "missing mask file")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
// get model config from context (middleware set it)
|
// get model config from context (middleware set it)
|
||||||
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
|
||||||
if !ok || cfg == nil {
|
if !ok || cfg == nil {
|
||||||
log.Error().Msg("Inpainting Endpoint - model config not found in context")
|
xlog.Error("Inpainting Endpoint - model config not found in context")
|
||||||
return echo.ErrBadRequest
|
return echo.ErrBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
tmpDir := appConfig.GeneratedContentDir
|
tmpDir := appConfig.GeneratedContentDir
|
||||||
// Ensure the directory exists
|
// Ensure the directory exists
|
||||||
if err := os.MkdirAll(tmpDir, 0750); err != nil {
|
if err := os.MkdirAll(tmpDir, 0750); err != nil {
|
||||||
log.Error().Err(err).Msgf("Inpainting Endpoint - failed to create generated content dir: %s", tmpDir)
|
xlog.Error("Inpainting Endpoint - failed to create generated content dir", "error", err, "dir", tmpDir)
|
||||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to prepare storage")
|
return echo.NewHTTPError(http.StatusInternalServerError, "failed to prepare storage")
|
||||||
}
|
}
|
||||||
id := uuid.New().String()
|
id := uuid.New().String()
|
||||||
@@ -132,32 +132,32 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
// Best-effort cleanup; log any failures
|
// Best-effort cleanup; log any failures
|
||||||
if jf != nil {
|
if jf != nil {
|
||||||
if cerr := jf.Close(); cerr != nil {
|
if cerr := jf.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file in cleanup")
|
xlog.Warn("Inpainting Endpoint - failed to close temp json file in cleanup", "error", cerr)
|
||||||
}
|
}
|
||||||
if name := jf.Name(); name != "" {
|
if name := jf.Name(); name != "" {
|
||||||
if rerr := os.Remove(name); rerr != nil && !os.IsNotExist(rerr) {
|
if rerr := os.Remove(name); rerr != nil && !os.IsNotExist(rerr) {
|
||||||
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove temp json file %s in cleanup", name)
|
xlog.Warn("Inpainting Endpoint - failed to remove temp json file in cleanup", "error", rerr, "file", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if jsonPath != "" {
|
if jsonPath != "" {
|
||||||
if rerr := os.Remove(jsonPath); rerr != nil && !os.IsNotExist(rerr) {
|
if rerr := os.Remove(jsonPath); rerr != nil && !os.IsNotExist(rerr) {
|
||||||
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove json file %s in cleanup", jsonPath)
|
xlog.Warn("Inpainting Endpoint - failed to remove json file in cleanup", "error", rerr, "file", jsonPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if dst != "" {
|
if dst != "" {
|
||||||
if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
|
if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
|
||||||
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove dst file %s in cleanup", dst)
|
xlog.Warn("Inpainting Endpoint - failed to remove dst file in cleanup", "error", rerr, "file", dst)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if origRef != "" {
|
if origRef != "" {
|
||||||
if rerr := os.Remove(origRef); rerr != nil && !os.IsNotExist(rerr) {
|
if rerr := os.Remove(origRef); rerr != nil && !os.IsNotExist(rerr) {
|
||||||
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove orig ref file %s in cleanup", origRef)
|
xlog.Warn("Inpainting Endpoint - failed to remove orig ref file in cleanup", "error", rerr, "file", origRef)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if maskRef != "" {
|
if maskRef != "" {
|
||||||
if rerr := os.Remove(maskRef); rerr != nil && !os.IsNotExist(rerr) {
|
if rerr := os.Remove(maskRef); rerr != nil && !os.IsNotExist(rerr) {
|
||||||
log.Warn().Err(rerr).Msgf("Inpainting Endpoint - failed to remove mask ref file %s in cleanup", maskRef)
|
xlog.Warn("Inpainting Endpoint - failed to remove mask ref file in cleanup", "error", rerr, "file", maskRef)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -175,7 +175,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if cerr := origTmp.Close(); cerr != nil {
|
if cerr := origTmp.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close orig temp file")
|
xlog.Warn("Inpainting Endpoint - failed to close orig temp file", "error", cerr)
|
||||||
}
|
}
|
||||||
origRef = origTmp.Name()
|
origRef = origTmp.Name()
|
||||||
|
|
||||||
@@ -192,19 +192,19 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if cerr := maskTmp.Close(); cerr != nil {
|
if cerr := maskTmp.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close mask temp file")
|
xlog.Warn("Inpainting Endpoint - failed to close mask temp file", "error", cerr)
|
||||||
}
|
}
|
||||||
maskRef = maskTmp.Name()
|
maskRef = maskTmp.Name()
|
||||||
// write JSON
|
// write JSON
|
||||||
enc := json.NewEncoder(jf)
|
enc := json.NewEncoder(jf)
|
||||||
if err := enc.Encode(jsonFile); err != nil {
|
if err := enc.Encode(jsonFile); err != nil {
|
||||||
if cerr := jf.Close(); cerr != nil {
|
if cerr := jf.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file after encode error")
|
xlog.Warn("Inpainting Endpoint - failed to close temp json file after encode error", "error", cerr)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if cerr := jf.Close(); cerr != nil {
|
if cerr := jf.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close temp json file")
|
xlog.Warn("Inpainting Endpoint - failed to close temp json file", "error", cerr)
|
||||||
}
|
}
|
||||||
// rename to desired name
|
// rename to desired name
|
||||||
if err := os.Rename(jf.Name(), jsonPath); err != nil {
|
if err := os.Rename(jf.Name(), jsonPath); err != nil {
|
||||||
@@ -216,7 +216,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if cerr := outTmp.Close(); cerr != nil {
|
if cerr := outTmp.Close(); cerr != nil {
|
||||||
log.Warn().Err(cerr).Msg("Inpainting Endpoint - failed to close out temp file")
|
xlog.Warn("Inpainting Endpoint - failed to close out temp file", "error", cerr)
|
||||||
}
|
}
|
||||||
dst = outTmp.Name() + ".png"
|
dst = outTmp.Name() + ".png"
|
||||||
if err := os.Rename(outTmp.Name(), dst); err != nil {
|
if err := os.Rename(outTmp.Name(), dst); err != nil {
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/templates"
|
"github.com/mudler/LocalAI/core/templates"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/cogito"
|
"github.com/mudler/cogito"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MCPCompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
|
// MCPCompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions
|
||||||
@@ -102,19 +102,19 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
|
|||||||
cogito.WithContext(ctxWithCancellation),
|
cogito.WithContext(ctxWithCancellation),
|
||||||
cogito.WithMCPs(sessions...),
|
cogito.WithMCPs(sessions...),
|
||||||
cogito.WithStatusCallback(func(s string) {
|
cogito.WithStatusCallback(func(s string) {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Status: %s", config.Name, s)
|
xlog.Debug("[model agent] Status", "model", config.Name, "status", s)
|
||||||
}),
|
}),
|
||||||
cogito.WithReasoningCallback(func(s string) {
|
cogito.WithReasoningCallback(func(s string) {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Reasoning: %s", config.Name, s)
|
xlog.Debug("[model agent] Reasoning", "model", config.Name, "reasoning", s)
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Tool call: %s, reasoning: %s, arguments: %+v", config.Name, t.Name, t.Reasoning, t.Arguments)
|
xlog.Debug("[model agent] Tool call", "model", config.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
|
||||||
return cogito.ToolCallDecision{
|
return cogito.ToolCallDecision{
|
||||||
Approved: true,
|
Approved: true,
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
||||||
log.Debug().Msgf("[model agent] [model: %s] Tool call result: %s, result: %s, tool arguments: %+v", config.Name, t.Name, t.Result, t.ToolArguments)
|
xlog.Debug("[model agent] Tool call result", "model", config.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -140,7 +140,7 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
|
|||||||
}
|
}
|
||||||
|
|
||||||
jsonResult, _ := json.Marshal(resp)
|
jsonResult, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", jsonResult)
|
xlog.Debug("Response", "response", string(jsonResult))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(200, resp)
|
return c.JSON(200, resp)
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -212,12 +212,12 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
return func(c *websocket.Conn) {
|
return func(c *websocket.Conn) {
|
||||||
|
|
||||||
evaluator := application.TemplatesEvaluator()
|
evaluator := application.TemplatesEvaluator()
|
||||||
log.Debug().Msgf("WebSocket connection established with '%s'", c.RemoteAddr().String())
|
xlog.Debug("WebSocket connection established", "address", c.RemoteAddr().String())
|
||||||
if intent != "transcription" {
|
if intent != "transcription" {
|
||||||
sendNotImplemented(c, "Only transcription mode is supported which requires the intent=transcription parameter")
|
sendNotImplemented(c, "Only transcription mode is supported which requires the intent=transcription parameter")
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Realtime params: model=%s, intent=%s", model, intent)
|
xlog.Debug("Realtime params", "model", model, "intent", intent)
|
||||||
|
|
||||||
sessionID := generateSessionID()
|
sessionID := generateSessionID()
|
||||||
session := &Session{
|
session := &Session{
|
||||||
@@ -265,7 +265,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
application.ApplicationConfig(),
|
application.ApplicationConfig(),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("failed to load model: %s", err.Error())
|
xlog.Error("failed to load model", "error", err)
|
||||||
sendError(c, "model_load_error", "Failed to load model", "", "")
|
sendError(c, "model_load_error", "Failed to load model", "", "")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -301,14 +301,14 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
if _, msg, err = c.ReadMessage(); err != nil {
|
if _, msg, err = c.ReadMessage(); err != nil {
|
||||||
log.Error().Msgf("read: %s", err.Error())
|
xlog.Error("read error", "error", err)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse the incoming message
|
// Parse the incoming message
|
||||||
var incomingMsg IncomingMessage
|
var incomingMsg IncomingMessage
|
||||||
if err := json.Unmarshal(msg, &incomingMsg); err != nil {
|
if err := json.Unmarshal(msg, &incomingMsg); err != nil {
|
||||||
log.Error().Msgf("invalid json: %s", err.Error())
|
xlog.Error("invalid json", "error", err)
|
||||||
sendError(c, "invalid_json", "Invalid JSON format", "", "")
|
sendError(c, "invalid_json", "Invalid JSON format", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -316,10 +316,10 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
var sessionUpdate types.ClientSession
|
var sessionUpdate types.ClientSession
|
||||||
switch incomingMsg.Type {
|
switch incomingMsg.Type {
|
||||||
case types.ClientEventTypeTranscriptionSessionUpdate:
|
case types.ClientEventTypeTranscriptionSessionUpdate:
|
||||||
log.Debug().Msgf("recv: %s", msg)
|
xlog.Debug("recv", "message", string(msg))
|
||||||
|
|
||||||
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
|
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
|
||||||
log.Error().Msgf("failed to unmarshal 'transcription_session.update': %s", err.Error())
|
xlog.Error("failed to unmarshal 'transcription_session.update'", "error", err)
|
||||||
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
|
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -330,7 +330,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
application.ModelLoader(),
|
application.ModelLoader(),
|
||||||
application.ApplicationConfig(),
|
application.ApplicationConfig(),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
log.Error().Msgf("failed to update session: %s", err.Error())
|
xlog.Error("failed to update session", "error", err)
|
||||||
sendError(c, "session_update_error", "Failed to update session", "", "")
|
sendError(c, "session_update_error", "Failed to update session", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -344,11 +344,11 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
})
|
})
|
||||||
|
|
||||||
case types.ClientEventTypeSessionUpdate:
|
case types.ClientEventTypeSessionUpdate:
|
||||||
log.Debug().Msgf("recv: %s", msg)
|
xlog.Debug("recv", "message", string(msg))
|
||||||
|
|
||||||
// Update session configurations
|
// Update session configurations
|
||||||
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
|
if err := json.Unmarshal(incomingMsg.Session, &sessionUpdate); err != nil {
|
||||||
log.Error().Msgf("failed to unmarshal 'session.update': %s", err.Error())
|
xlog.Error("failed to unmarshal 'session.update'", "error", err)
|
||||||
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
|
sendError(c, "invalid_session_update", "Invalid session update format", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -359,7 +359,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
application.ModelLoader(),
|
application.ModelLoader(),
|
||||||
application.ApplicationConfig(),
|
application.ApplicationConfig(),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
log.Error().Msgf("failed to update session: %s", err.Error())
|
xlog.Error("failed to update session", "error", err)
|
||||||
sendError(c, "session_update_error", "Failed to update session", "", "")
|
sendError(c, "session_update_error", "Failed to update session", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -373,7 +373,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
})
|
})
|
||||||
|
|
||||||
if session.TurnDetection.Type == types.ServerTurnDetectionTypeServerVad && !vadServerStarted {
|
if session.TurnDetection.Type == types.ServerTurnDetectionTypeServerVad && !vadServerStarted {
|
||||||
log.Debug().Msg("Starting VAD goroutine...")
|
xlog.Debug("Starting VAD goroutine...")
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
@@ -382,7 +382,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
}()
|
}()
|
||||||
vadServerStarted = true
|
vadServerStarted = true
|
||||||
} else if session.TurnDetection.Type != types.ServerTurnDetectionTypeServerVad && vadServerStarted {
|
} else if session.TurnDetection.Type != types.ServerTurnDetectionTypeServerVad && vadServerStarted {
|
||||||
log.Debug().Msg("Stopping VAD goroutine...")
|
xlog.Debug("Stopping VAD goroutine...")
|
||||||
|
|
||||||
wg.Add(-1)
|
wg.Add(-1)
|
||||||
go func() {
|
go func() {
|
||||||
@@ -393,7 +393,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
case types.ClientEventTypeInputAudioBufferAppend:
|
case types.ClientEventTypeInputAudioBufferAppend:
|
||||||
// Handle 'input_audio_buffer.append'
|
// Handle 'input_audio_buffer.append'
|
||||||
if incomingMsg.Audio == "" {
|
if incomingMsg.Audio == "" {
|
||||||
log.Error().Msg("Audio data is missing in 'input_audio_buffer.append'")
|
xlog.Error("Audio data is missing in 'input_audio_buffer.append'")
|
||||||
sendError(c, "missing_audio_data", "Audio data is missing", "", "")
|
sendError(c, "missing_audio_data", "Audio data is missing", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -401,7 +401,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
// Decode base64 audio data
|
// Decode base64 audio data
|
||||||
decodedAudio, err := base64.StdEncoding.DecodeString(incomingMsg.Audio)
|
decodedAudio, err := base64.StdEncoding.DecodeString(incomingMsg.Audio)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("failed to decode audio data: %s", err.Error())
|
xlog.Error("failed to decode audio data", "error", err)
|
||||||
sendError(c, "invalid_audio_data", "Failed to decode audio data", "", "")
|
sendError(c, "invalid_audio_data", "Failed to decode audio data", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -412,7 +412,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
session.AudioBufferLock.Unlock()
|
session.AudioBufferLock.Unlock()
|
||||||
|
|
||||||
case types.ClientEventTypeInputAudioBufferCommit:
|
case types.ClientEventTypeInputAudioBufferCommit:
|
||||||
log.Debug().Msgf("recv: %s", msg)
|
xlog.Debug("recv", "message", string(msg))
|
||||||
|
|
||||||
// TODO: Trigger transcription.
|
// TODO: Trigger transcription.
|
||||||
// TODO: Ignore this if VAD enabled or interrupt VAD?
|
// TODO: Ignore this if VAD enabled or interrupt VAD?
|
||||||
@@ -458,12 +458,12 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
})
|
})
|
||||||
|
|
||||||
case types.ClientEventTypeConversationItemCreate:
|
case types.ClientEventTypeConversationItemCreate:
|
||||||
log.Debug().Msgf("recv: %s", msg)
|
xlog.Debug("recv", "message", string(msg))
|
||||||
|
|
||||||
// Handle creating new conversation items
|
// Handle creating new conversation items
|
||||||
var item types.ConversationItemCreateEvent
|
var item types.ConversationItemCreateEvent
|
||||||
if err := json.Unmarshal(incomingMsg.Item, &item); err != nil {
|
if err := json.Unmarshal(incomingMsg.Item, &item); err != nil {
|
||||||
log.Error().Msgf("failed to unmarshal 'conversation.item.create': %s", err.Error())
|
xlog.Error("failed to unmarshal 'conversation.item.create'", "error", err)
|
||||||
sendError(c, "invalid_item", "Invalid item format", "", "")
|
sendError(c, "invalid_item", "Invalid item format", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -494,7 +494,7 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
var responseCreate types.ResponseCreateEvent
|
var responseCreate types.ResponseCreateEvent
|
||||||
if len(incomingMsg.Response) > 0 {
|
if len(incomingMsg.Response) > 0 {
|
||||||
if err := json.Unmarshal(incomingMsg.Response, &responseCreate); err != nil {
|
if err := json.Unmarshal(incomingMsg.Response, &responseCreate); err != nil {
|
||||||
log.Error().Msgf("failed to unmarshal 'response.create' response object: %s", err.Error())
|
xlog.Error("failed to unmarshal 'response.create' response object", "error", err)
|
||||||
sendError(c, "invalid_response_create", "Invalid response create format", "", "")
|
sendError(c, "invalid_response_create", "Invalid response create format", "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -515,14 +515,14 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
// }()
|
// }()
|
||||||
|
|
||||||
case types.ClientEventTypeResponseCancel:
|
case types.ClientEventTypeResponseCancel:
|
||||||
log.Printf("recv: %s", msg)
|
xlog.Debug("recv", "message", string(msg))
|
||||||
|
|
||||||
// Handle cancellation of ongoing responses
|
// Handle cancellation of ongoing responses
|
||||||
// Implement cancellation logic as needed
|
// Implement cancellation logic as needed
|
||||||
sendNotImplemented(c, "response.cancel")
|
sendNotImplemented(c, "response.cancel")
|
||||||
|
|
||||||
default:
|
default:
|
||||||
log.Error().Msgf("unknown message type: %s", incomingMsg.Type)
|
xlog.Error("unknown message type", "type", incomingMsg.Type)
|
||||||
sendError(c, "unknown_message_type", fmt.Sprintf("Unknown message type: %s", incomingMsg.Type), "", "")
|
sendError(c, "unknown_message_type", fmt.Sprintf("Unknown message type: %s", incomingMsg.Type), "", "")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -542,11 +542,11 @@ func registerRealtime(application *application.Application, model, intent string
|
|||||||
func sendEvent(c *websocket.Conn, event types.ServerEvent) {
|
func sendEvent(c *websocket.Conn, event types.ServerEvent) {
|
||||||
eventBytes, err := json.Marshal(event)
|
eventBytes, err := json.Marshal(event)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("failed to marshal event: %s", err.Error())
|
xlog.Error("failed to marshal event", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = c.WriteMessage(websocket.TextMessage, eventBytes); err != nil {
|
if err = c.WriteMessage(websocket.TextMessage, eventBytes); err != nil {
|
||||||
log.Error().Msgf("write: %s", err.Error())
|
xlog.Error("write error", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -681,10 +681,10 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
|
|||||||
segments, err := runVAD(vadContext, session, aints)
|
segments, err := runVAD(vadContext, session, aints)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err.Error() == "unexpected speech end" {
|
if err.Error() == "unexpected speech end" {
|
||||||
log.Debug().Msg("VAD cancelled")
|
xlog.Debug("VAD cancelled")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
log.Error().Msgf("failed to process audio: %s", err.Error())
|
xlog.Error("failed to process audio", "error", err)
|
||||||
sendError(c, "processing_error", "Failed to process audio: "+err.Error(), "", "")
|
sendError(c, "processing_error", "Failed to process audio: "+err.Error(), "", "")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -697,7 +697,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
|
|||||||
session.AudioBufferLock.Lock()
|
session.AudioBufferLock.Lock()
|
||||||
session.InputAudioBuffer = nil
|
session.InputAudioBuffer = nil
|
||||||
session.AudioBufferLock.Unlock()
|
session.AudioBufferLock.Unlock()
|
||||||
log.Debug().Msgf("Detected silence for a while, clearing audio buffer")
|
xlog.Debug("Detected silence for a while, clearing audio buffer")
|
||||||
|
|
||||||
sendEvent(c, types.InputAudioBufferClearedEvent{
|
sendEvent(c, types.InputAudioBufferClearedEvent{
|
||||||
ServerEventBase: types.ServerEventBase{
|
ServerEventBase: types.ServerEventBase{
|
||||||
@@ -729,7 +729,7 @@ func handleVAD(cfg *config.ModelConfig, evaluator *templates.Evaluator, session
|
|||||||
}
|
}
|
||||||
|
|
||||||
if float32(audioLength)-segEndTime > float32(silenceThreshold) {
|
if float32(audioLength)-segEndTime > float32(silenceThreshold) {
|
||||||
log.Debug().Msgf("Detected end of speech segment")
|
xlog.Debug("Detected end of speech segment")
|
||||||
session.AudioBufferLock.Lock()
|
session.AudioBufferLock.Lock()
|
||||||
session.InputAudioBuffer = nil
|
session.InputAudioBuffer = nil
|
||||||
session.AudioBufferLock.Unlock()
|
session.AudioBufferLock.Unlock()
|
||||||
@@ -769,21 +769,21 @@ func commitUtterance(ctx context.Context, utt []byte, cfg *config.ModelConfig, e
|
|||||||
|
|
||||||
f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav")
|
f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("failed to create temp file: %s", err.Error())
|
xlog.Error("failed to create temp file", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
defer os.Remove(f.Name())
|
defer os.Remove(f.Name())
|
||||||
log.Debug().Msgf("Writing to %s\n", f.Name())
|
xlog.Debug("Writing to file", "file", f.Name())
|
||||||
|
|
||||||
hdr := laudio.NewWAVHeader(uint32(len(utt)))
|
hdr := laudio.NewWAVHeader(uint32(len(utt)))
|
||||||
if err := hdr.Write(f); err != nil {
|
if err := hdr.Write(f); err != nil {
|
||||||
log.Error().Msgf("Failed to write WAV header: %s", err.Error())
|
xlog.Error("Failed to write WAV header", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := f.Write(utt); err != nil {
|
if _, err := f.Write(utt); err != nil {
|
||||||
log.Error().Msgf("Failed to write audio data: %s", err.Error())
|
xlog.Error("Failed to write audio data", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1106,14 +1106,14 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
|
|||||||
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
|
textContentToReturn = functions.ParseTextContent(s, config.FunctionsConfig)
|
||||||
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
|
s = functions.CleanupLLMResult(s, config.FunctionsConfig)
|
||||||
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
|
results := functions.ParseFunctionCall(s, config.FunctionsConfig)
|
||||||
log.Debug().Msgf("Text content to return: %s", textContentToReturn)
|
xlog.Debug("Text content to return", "text", textContentToReturn)
|
||||||
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
|
noActionsToRun := len(results) > 0 && results[0].Name == noActionName || len(results) == 0
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case noActionsToRun:
|
case noActionsToRun:
|
||||||
result, err := handleQuestion(config, input, ml, startupOptions, results, s, predInput)
|
result, err := handleQuestion(config, input, ml, startupOptions, results, s, predInput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error handling question")
|
xlog.Error("error handling question", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
*c = append(*c, schema.Choice{
|
*c = append(*c, schema.Choice{
|
||||||
@@ -1187,7 +1187,7 @@ func processTextResponse(config *config.ModelConfig, session *Session, prompt st
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
respData, _ := json.Marshal(resp)
|
respData, _ := json.Marshal(resp)
|
||||||
log.Debug().Msgf("Response: %s", respData)
|
xlog.Debug("Response", "response", string(respData))
|
||||||
|
|
||||||
// Return the prediction in the response body
|
// Return the prediction in the response body
|
||||||
return c.JSON(resp)
|
return c.JSON(resp)
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
grpcClient "github.com/mudler/LocalAI/pkg/grpc"
|
grpcClient "github.com/mudler/LocalAI/pkg/grpc"
|
||||||
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
model "github.com/mudler/LocalAI/pkg/model"
|
model "github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -209,7 +209,7 @@ func newModel(pipeline *config.Pipeline, cl *config.ModelConfigLoader, ml *model
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msg("Loading a wrapped model")
|
xlog.Debug("Loading a wrapped model")
|
||||||
|
|
||||||
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
|
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
|
||||||
cfgLLM, err := cl.LoadModelConfigFileByName(pipeline.LLM, ml.ModelPath)
|
cfgLLM, err := cl.LoadModelConfigFileByName(pipeline.LLM, ml.ModelPath)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
model "github.com/mudler/LocalAI/pkg/model"
|
model "github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create
|
// TranscriptEndpoint is the OpenAI Whisper API endpoint https://platform.openai.com/docs/api-reference/audio/create
|
||||||
@@ -64,18 +64,18 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
|
|||||||
}
|
}
|
||||||
|
|
||||||
if _, err := io.Copy(dstFile, f); err != nil {
|
if _, err := io.Copy(dstFile, f); err != nil {
|
||||||
log.Debug().Msgf("Audio file copying error %+v - %+v - err %+v", file.Filename, dst, err)
|
xlog.Debug("Audio file copying error", "filename", file.Filename, "dst", dst, "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Audio file copied to: %+v", dst)
|
xlog.Debug("Audio file copied", "dst", dst)
|
||||||
|
|
||||||
tr, err := backend.ModelTranscription(dst, input.Language, input.Translate, diarize, prompt, ml, *config, appConfig)
|
tr, err := backend.ModelTranscription(dst, input.Language, input.Translate, diarize, prompt, ml, *config, appConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Trascribed: %+v", tr)
|
xlog.Debug("Transcribed", "transcription", tr)
|
||||||
// TODO: handle different outputs here
|
// TODO: handle different outputs here
|
||||||
return c.JSON(http.StatusOK, tr)
|
return c.JSON(http.StatusOK, tr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/explorer"
|
"github.com/mudler/LocalAI/core/explorer"
|
||||||
"github.com/mudler/LocalAI/core/http/middleware"
|
"github.com/mudler/LocalAI/core/http/middleware"
|
||||||
"github.com/mudler/LocalAI/core/http/routes"
|
"github.com/mudler/LocalAI/core/http/routes"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Explorer(db *explorer.Database) *echo.Echo {
|
func Explorer(db *explorer.Database) *echo.Echo {
|
||||||
@@ -37,7 +37,7 @@ func Explorer(db *explorer.Database) *echo.Echo {
|
|||||||
staticFS, err := fs.Sub(embedDirStatic, "static")
|
staticFS, err := fs.Sub(embedDirStatic, "static")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Log error but continue - static files might not work
|
// Log error but continue - static files might not work
|
||||||
log.Error().Err(err).Msg("failed to create static filesystem")
|
xlog.Error("failed to create static filesystem", "error", err)
|
||||||
} else {
|
} else {
|
||||||
e.StaticFS("/static", staticFS)
|
e.StaticFS("/static", staticFS)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/functions"
|
"github.com/mudler/LocalAI/pkg/functions"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type correlationIDKeyType string
|
type correlationIDKeyType string
|
||||||
@@ -82,7 +82,7 @@ func (re *RequestExtractor) BuildConstantDefaultModelNameMiddleware(defaultModel
|
|||||||
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
||||||
if !ok || localModelName == "" {
|
if !ok || localModelName == "" {
|
||||||
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, defaultModelName)
|
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, defaultModelName)
|
||||||
log.Debug().Str("defaultModelName", defaultModelName).Msg("context local model name not found, setting to default")
|
xlog.Debug("context local model name not found, setting to default", "defaultModelName", defaultModelName)
|
||||||
}
|
}
|
||||||
return next(c)
|
return next(c)
|
||||||
}
|
}
|
||||||
@@ -100,19 +100,19 @@ func (re *RequestExtractor) BuildFilteredFirstAvailableDefaultModel(filterFn con
|
|||||||
|
|
||||||
modelNames, err := services.ListModels(re.modelConfigLoader, re.modelLoader, filterFn, services.SKIP_IF_CONFIGURED)
|
modelNames, err := services.ListModels(re.modelConfigLoader, re.modelLoader, filterFn, services.SKIP_IF_CONFIGURED)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()")
|
xlog.Error("non-fatal error calling ListModels during SetDefaultModelNameToFirstAvailable()", "error", err)
|
||||||
return next(c)
|
return next(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(modelNames) == 0 {
|
if len(modelNames) == 0 {
|
||||||
log.Warn().Msg("SetDefaultModelNameToFirstAvailable used with no matching models installed")
|
xlog.Warn("SetDefaultModelNameToFirstAvailable used with no matching models installed")
|
||||||
// This is non-fatal - making it so was breaking the case of direct installation of raw models
|
// This is non-fatal - making it so was breaking the case of direct installation of raw models
|
||||||
// return errors.New("this endpoint requires at least one model to be installed")
|
// return errors.New("this endpoint requires at least one model to be installed")
|
||||||
return next(c)
|
return next(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, modelNames[0])
|
c.Set(CONTEXT_LOCALS_KEY_MODEL_NAME, modelNames[0])
|
||||||
log.Debug().Str("first model name", modelNames[0]).Msg("context local model name not found, setting to the first model")
|
xlog.Debug("context local model name not found, setting to the first model", "first model name", modelNames[0])
|
||||||
return next(c)
|
return next(c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -135,7 +135,7 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
|
|||||||
if input.ModelName(nil) == "" {
|
if input.ModelName(nil) == "" {
|
||||||
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
localModelName, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_NAME).(string)
|
||||||
if ok && localModelName != "" {
|
if ok && localModelName != "" {
|
||||||
log.Debug().Str("context localModelName", localModelName).Msg("overriding empty model name in request body with value found earlier in middleware chain")
|
xlog.Debug("overriding empty model name in request body with value found earlier in middleware chain", "context localModelName", localModelName)
|
||||||
input.ModelName(&localModelName)
|
input.ModelName(&localModelName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,10 +143,9 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
|
|||||||
cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig)
|
cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(input.ModelName(nil), re.applicationConfig)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Err(err)
|
xlog.Warn("Model Configuration File not found", "model", input.ModelName(nil), "error", err)
|
||||||
log.Warn().Msgf("Model Configuration File not found for %q", input.ModelName(nil))
|
|
||||||
} else if cfg.Model == "" && input.ModelName(nil) != "" {
|
} else if cfg.Model == "" && input.ModelName(nil) != "" {
|
||||||
log.Debug().Str("input.ModelName", input.ModelName(nil)).Msg("config does not include model, using input")
|
xlog.Debug("config does not include model, using input", "input.ModelName", input.ModelName(nil))
|
||||||
cfg.Model = input.ModelName(nil)
|
cfg.Model = input.ModelName(nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,7 +202,7 @@ func (re *RequestExtractor) SetOpenAIRequest(c echo.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Model == "" {
|
if cfg.Model == "" {
|
||||||
log.Debug().Str("input.Model", input.Model).Msg("replacing empty cfg.Model with input value")
|
xlog.Debug("replacing empty cfg.Model with input value", "input.Model", input.Model)
|
||||||
cfg.Model = input.Model
|
cfg.Model = input.Model
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -331,7 +330,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
|
|||||||
// Decode content as base64 either if it's an URL or base64 text
|
// Decode content as base64 either if it's an URL or base64 text
|
||||||
base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL)
|
base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("Failed encoding video: %s", err)
|
xlog.Error("Failed encoding video", "error", err)
|
||||||
continue CONTENT
|
continue CONTENT
|
||||||
}
|
}
|
||||||
input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff
|
input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff
|
||||||
@@ -341,7 +340,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
|
|||||||
// Decode content as base64 either if it's an URL or base64 text
|
// Decode content as base64 either if it's an URL or base64 text
|
||||||
base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL)
|
base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("Failed encoding audio: %s", err)
|
xlog.Error("Failed encoding audio", "error", err)
|
||||||
continue CONTENT
|
continue CONTENT
|
||||||
}
|
}
|
||||||
input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff
|
input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff
|
||||||
@@ -356,7 +355,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
|
|||||||
// Decode content as base64 either if it's an URL or base64 text
|
// Decode content as base64 either if it's an URL or base64 text
|
||||||
base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL)
|
base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Msgf("Failed encoding image: %s", err)
|
xlog.Error("Failed encoding image", "error", err)
|
||||||
continue CONTENT
|
continue CONTENT
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -410,7 +409,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
|
|||||||
config.TypicalP = input.TypicalP
|
config.TypicalP = input.TypicalP
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("input.Input", fmt.Sprintf("%+v", input.Input))
|
xlog.Debug("input.Input", "input", fmt.Sprintf("%+v", input.Input))
|
||||||
|
|
||||||
switch inputs := input.Input.(type) {
|
switch inputs := input.Input.(type) {
|
||||||
case string:
|
case string:
|
||||||
@@ -434,7 +433,7 @@ func mergeOpenAIRequestAndModelConfig(config *config.ModelConfig, input *schema.
|
|||||||
case string:
|
case string:
|
||||||
inputStrings = append(inputStrings, ii)
|
inputStrings = append(inputStrings, ii)
|
||||||
default:
|
default:
|
||||||
log.Error().Msgf("Unknown input type: %T", ii)
|
xlog.Error("Unknown input type", "type", fmt.Sprintf("%T", ii))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
config.InputToken = append(config.InputToken, tokens)
|
config.InputToken = append(config.InputToken, tokens)
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/services"
|
"github.com/mudler/LocalAI/core/services"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -143,11 +143,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
// Cancel operation endpoint
|
// Cancel operation endpoint
|
||||||
app.POST("/api/operations/:jobID/cancel", func(c echo.Context) error {
|
app.POST("/api/operations/:jobID/cancel", func(c echo.Context) error {
|
||||||
jobID := c.Param("jobID")
|
jobID := c.Param("jobID")
|
||||||
log.Debug().Msgf("API request to cancel operation: %s", jobID)
|
xlog.Debug("API request to cancel operation", "jobID", jobID)
|
||||||
|
|
||||||
err := galleryService.CancelOperation(jobID)
|
err := galleryService.CancelOperation(jobID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed to cancel operation: %s", jobID)
|
xlog.Error("Failed to cancel operation", "error", err, "jobID", jobID)
|
||||||
return c.JSON(http.StatusBadRequest, map[string]interface{}{
|
return c.JSON(http.StatusBadRequest, map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
@@ -176,7 +176,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
|
|
||||||
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
|
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("could not list models from galleries")
|
xlog.Error("could not list models from galleries", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
@@ -246,7 +246,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
|
|
||||||
// Skip duplicate IDs to prevent Alpine.js x-for errors
|
// Skip duplicate IDs to prevent Alpine.js x-for errors
|
||||||
if seenIDs[modelID] {
|
if seenIDs[modelID] {
|
||||||
log.Debug().Msgf("Skipping duplicate model ID: %s", modelID)
|
xlog.Debug("Skipping duplicate model ID", "modelID", modelID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seenIDs[modelID] = true
|
seenIDs[modelID] = true
|
||||||
@@ -320,7 +320,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid model ID",
|
"error": "invalid model ID",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API job submitted to install: %+v\n", galleryID)
|
xlog.Debug("API job submitted to install", "galleryID", galleryID)
|
||||||
|
|
||||||
id, err := uuid.NewUUID()
|
id, err := uuid.NewUUID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -362,7 +362,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid model ID",
|
"error": "invalid model ID",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API job submitted to delete: %+v\n", galleryID)
|
xlog.Debug("API job submitted to delete", "galleryID", galleryID)
|
||||||
|
|
||||||
var galleryName = galleryID
|
var galleryName = galleryID
|
||||||
if strings.Contains(galleryID, "@") {
|
if strings.Contains(galleryID, "@") {
|
||||||
@@ -412,7 +412,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid model ID",
|
"error": "invalid model ID",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API job submitted to get config for: %+v\n", galleryID)
|
xlog.Debug("API job submitted to get config", "galleryID", galleryID)
|
||||||
|
|
||||||
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
|
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.SystemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -498,7 +498,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
|
|
||||||
backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
|
backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("could not list backends from galleries")
|
xlog.Error("could not list backends from galleries", "error", err)
|
||||||
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
@@ -568,7 +568,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
|
|
||||||
// Skip duplicate IDs to prevent Alpine.js x-for errors
|
// Skip duplicate IDs to prevent Alpine.js x-for errors
|
||||||
if seenBackendIDs[backendID] {
|
if seenBackendIDs[backendID] {
|
||||||
log.Debug().Msgf("Skipping duplicate backend ID: %s", backendID)
|
xlog.Debug("Skipping duplicate backend ID", "backendID", backendID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seenBackendIDs[backendID] = true
|
seenBackendIDs[backendID] = true
|
||||||
@@ -640,7 +640,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid backend ID",
|
"error": "invalid backend ID",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API job submitted to install backend: %+v\n", backendID)
|
xlog.Debug("API job submitted to install backend", "backendID", backendID)
|
||||||
|
|
||||||
id, err := uuid.NewUUID()
|
id, err := uuid.NewUUID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -695,7 +695,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("uri", req.URI).Str("name", req.Name).Str("alias", req.Alias).Msg("API job submitted to install external backend")
|
xlog.Debug("API job submitted to install external backend", "uri", req.URI, "name", req.Name, "alias", req.Alias)
|
||||||
|
|
||||||
id, err := uuid.NewUUID()
|
id, err := uuid.NewUUID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -745,7 +745,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid backend ID",
|
"error": "invalid backend ID",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API job submitted to delete backend: %+v\n", backendID)
|
xlog.Debug("API job submitted to delete backend", "backendID", backendID)
|
||||||
|
|
||||||
var backendName = backendID
|
var backendName = backendID
|
||||||
if strings.Contains(backendID, "@") {
|
if strings.Contains(backendID, "@") {
|
||||||
@@ -831,11 +831,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
|||||||
"error": "invalid backend name",
|
"error": "invalid backend name",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("API request to delete system backend: %+v\n", backendName)
|
xlog.Debug("API request to delete system backend", "backendName", backendName)
|
||||||
|
|
||||||
// Use the gallery package to delete the backend
|
// Use the gallery package to delete the backend
|
||||||
if err := gallery.DeleteBackendFromSystem(appConfig.SystemState, backendName); err != nil {
|
if err := gallery.DeleteBackendFromSystem(appConfig.SystemState, backendName); err != nil {
|
||||||
log.Error().Err(err).Msgf("Failed to delete backend: %s", backendName)
|
xlog.Error("Failed to delete backend", "error", err, "backendName", backendName)
|
||||||
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const FederatedID = "federated"
|
const FederatedID = "federated"
|
||||||
@@ -43,7 +43,7 @@ func (fs *FederatedServer) RandomServer() string {
|
|||||||
tunnelAddresses = append(tunnelAddresses, v.ID)
|
tunnelAddresses = append(tunnelAddresses, v.ID)
|
||||||
} else {
|
} else {
|
||||||
delete(fs.requestTable, v.ID) // make sure it's not tracked
|
delete(fs.requestTable, v.ID) // make sure it's not tracked
|
||||||
log.Info().Msgf("Node %s is offline", v.ID)
|
xlog.Info("Node is offline", "node", v.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
|
|||||||
fs.Lock()
|
fs.Lock()
|
||||||
defer fs.Unlock()
|
defer fs.Unlock()
|
||||||
|
|
||||||
log.Debug().Any("request_table", fs.requestTable).Msgf("SelectLeastUsedServer()")
|
xlog.Debug("SelectLeastUsedServer()", "request_table", fs.requestTable)
|
||||||
|
|
||||||
// cycle over requestTable and find the entry with the lower number
|
// cycle over requestTable and find the entry with the lower number
|
||||||
// if there are multiple entries with the same number, select one randomly
|
// if there are multiple entries with the same number, select one randomly
|
||||||
@@ -93,7 +93,7 @@ func (fs *FederatedServer) SelectLeastUsedServer() string {
|
|||||||
minKey = k
|
minKey = k
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Debug().Any("requests_served", min).Any("request_table", fs.requestTable).Msgf("Selected tunnel %s", minKey)
|
xlog.Debug("Selected tunnel", "tunnel", minKey, "requests_served", min, "request_table", fs.requestTable)
|
||||||
|
|
||||||
return minKey
|
return minKey
|
||||||
}
|
}
|
||||||
@@ -104,7 +104,7 @@ func (fs *FederatedServer) RecordRequest(nodeID string) {
|
|||||||
// increment the counter for the nodeID in the requestTable
|
// increment the counter for the nodeID in the requestTable
|
||||||
fs.requestTable[nodeID]++
|
fs.requestTable[nodeID]++
|
||||||
|
|
||||||
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Recording request")
|
xlog.Debug("Recording request", "request_table", fs.requestTable, "request", nodeID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fs *FederatedServer) ensureRecordExist(nodeID string) {
|
func (fs *FederatedServer) ensureRecordExist(nodeID string) {
|
||||||
@@ -114,5 +114,5 @@ func (fs *FederatedServer) ensureRecordExist(nodeID string) {
|
|||||||
fs.requestTable[nodeID] = 0
|
fs.requestTable[nodeID] = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Any("request_table", fs.requestTable).Any("request", nodeID).Msgf("Ensure record exists")
|
xlog.Debug("Ensure record exists", "request_table", fs.requestTable, "request", nodeID)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/edgevpn/pkg/node"
|
"github.com/mudler/edgevpn/pkg/node"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (f *FederatedServer) Start(ctx context.Context) error {
|
func (f *FederatedServer) Start(ctx context.Context) error {
|
||||||
@@ -23,7 +23,7 @@ func (f *FederatedServer) Start(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := ServiceDiscoverer(ctx, n, f.p2ptoken, f.service, func(servicesID string, tunnel schema.NodeData) {
|
if err := ServiceDiscoverer(ctx, n, f.p2ptoken, f.service, func(servicesID string, tunnel schema.NodeData) {
|
||||||
log.Debug().Msgf("Discovered node: %s", tunnel.ID)
|
xlog.Debug("Discovered node", "node", tunnel.ID)
|
||||||
}, false); err != nil {
|
}, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -33,11 +33,11 @@ func (f *FederatedServer) Start(ctx context.Context) error {
|
|||||||
|
|
||||||
func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
|
func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
|
||||||
|
|
||||||
log.Info().Msgf("Allocating service '%s' on: %s", fs.service, fs.listenAddr)
|
xlog.Info("Allocating service", "service", fs.service, "address", fs.listenAddr)
|
||||||
// Open local port for listening
|
// Open local port for listening
|
||||||
l, err := net.Listen("tcp", fs.listenAddr)
|
l, err := net.Listen("tcp", fs.listenAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("Error listening")
|
xlog.Error("Error listening", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return errors.New("context canceled")
|
return errors.New("context canceled")
|
||||||
default:
|
default:
|
||||||
log.Debug().Msgf("New connection from %s", l.Addr().String())
|
xlog.Debug("New connection", "address", l.Addr().String())
|
||||||
// Listen for an incoming connection.
|
// Listen for an incoming connection.
|
||||||
conn, err := l.Accept()
|
conn, err := l.Accept()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,11 +68,11 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
|
|||||||
if fs.workerTarget != "" {
|
if fs.workerTarget != "" {
|
||||||
workerID = fs.workerTarget
|
workerID = fs.workerTarget
|
||||||
} else if fs.loadBalanced {
|
} else if fs.loadBalanced {
|
||||||
log.Debug().Msgf("Load balancing request")
|
xlog.Debug("Load balancing request")
|
||||||
|
|
||||||
workerID = fs.SelectLeastUsedServer()
|
workerID = fs.SelectLeastUsedServer()
|
||||||
if workerID == "" {
|
if workerID == "" {
|
||||||
log.Debug().Msgf("Least used server not found, selecting random")
|
xlog.Debug("Least used server not found, selecting random")
|
||||||
workerID = fs.RandomServer()
|
workerID = fs.RandomServer()
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -80,15 +80,15 @@ func (fs *FederatedServer) proxy(ctx context.Context, node *node.Node) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if workerID == "" {
|
if workerID == "" {
|
||||||
log.Error().Msg("No available nodes yet")
|
xlog.Error("No available nodes yet")
|
||||||
fs.sendHTMLResponse(conn, 503, "Sorry, waiting for nodes to connect")
|
fs.sendHTMLResponse(conn, 503, "Sorry, waiting for nodes to connect")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Selected node %s", workerID)
|
xlog.Debug("Selected node", "node", workerID)
|
||||||
nodeData, exists := GetNode(fs.service, workerID)
|
nodeData, exists := GetNode(fs.service, workerID)
|
||||||
if !exists {
|
if !exists {
|
||||||
log.Error().Msgf("Node %s not found", workerID)
|
xlog.Error("Node not found", "node", workerID)
|
||||||
fs.sendHTMLResponse(conn, 404, "Node not found")
|
fs.sendHTMLResponse(conn, 404, "Node not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -123,7 +123,7 @@ func (fs *FederatedServer) sendHTMLResponse(conn net.Conn, statusCode int, messa
|
|||||||
// Write the response to the client connection.
|
// Write the response to the client connection.
|
||||||
_, writeErr := io.WriteString(conn, response)
|
_, writeErr := io.WriteString(conn, response)
|
||||||
if writeErr != nil {
|
if writeErr != nil {
|
||||||
log.Error().Err(writeErr).Msg("Error writing response to client")
|
xlog.Error("Error writing response to client", "error", writeErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
"github.com/mudler/edgevpn/pkg/services"
|
"github.com/mudler/edgevpn/pkg/services"
|
||||||
"github.com/mudler/edgevpn/pkg/types"
|
"github.com/mudler/edgevpn/pkg/types"
|
||||||
eutils "github.com/mudler/edgevpn/pkg/utils"
|
eutils "github.com/mudler/edgevpn/pkg/utils"
|
||||||
|
zlog "github.com/mudler/xlog"
|
||||||
"github.com/multiformats/go-multiaddr"
|
"github.com/multiformats/go-multiaddr"
|
||||||
"github.com/phayes/freeport"
|
"github.com/phayes/freeport"
|
||||||
zlog "github.com/rs/zerolog/log"
|
|
||||||
|
|
||||||
"github.com/mudler/edgevpn/pkg/logger"
|
"github.com/mudler/edgevpn/pkg/logger"
|
||||||
)
|
)
|
||||||
@@ -94,7 +94,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
|
|||||||
existingValue.Unmarshal(service)
|
existingValue.Unmarshal(service)
|
||||||
// If mismatch, update the blockchain
|
// If mismatch, update the blockchain
|
||||||
if !found {
|
if !found {
|
||||||
zlog.Error().Msg("Service not found on blockchain")
|
zlog.Error("Service not found on blockchain")
|
||||||
conn.Close()
|
conn.Close()
|
||||||
// ll.Debugf("service '%s' not found on blockchain", serviceID)
|
// ll.Debugf("service '%s' not found on blockchain", serviceID)
|
||||||
return
|
return
|
||||||
@@ -103,7 +103,7 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
|
|||||||
// Decode the Peer
|
// Decode the Peer
|
||||||
d, err := peer.Decode(service.PeerID)
|
d, err := peer.Decode(service.PeerID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zlog.Error().Msg("cannot decode peer")
|
zlog.Error("cannot decode peer")
|
||||||
|
|
||||||
conn.Close()
|
conn.Close()
|
||||||
// ll.Debugf("could not decode peer '%s'", service.PeerID)
|
// ll.Debugf("could not decode peer '%s'", service.PeerID)
|
||||||
@@ -113,14 +113,14 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
|
|||||||
// Open a stream
|
// Open a stream
|
||||||
stream, err := node.Host().NewStream(ctx, d, protocol.ServiceProtocol.ID())
|
stream, err := node.Host().NewStream(ctx, d, protocol.ServiceProtocol.ID())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zlog.Error().Err(err).Msg("cannot open stream peer")
|
zlog.Error("cannot open stream peer", "error", err)
|
||||||
|
|
||||||
conn.Close()
|
conn.Close()
|
||||||
// ll.Debugf("could not open stream '%s'", err.Error())
|
// ll.Debugf("could not open stream '%s'", err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// ll.Debugf("(service %s) Redirecting", serviceID, l.Addr().String())
|
// ll.Debugf("(service %s) Redirecting", serviceID, l.Addr().String())
|
||||||
zlog.Info().Msgf("Redirecting %s to %s", conn.LocalAddr().String(), stream.Conn().RemoteMultiaddr().String())
|
zlog.Info("Redirecting", "from", conn.LocalAddr().String(), "to", stream.Conn().RemoteMultiaddr().String())
|
||||||
closer := make(chan struct{}, 2)
|
closer := make(chan struct{}, 2)
|
||||||
go copyStream(closer, stream, conn)
|
go copyStream(closer, stream, conn)
|
||||||
go copyStream(closer, conn, stream)
|
go copyStream(closer, conn, stream)
|
||||||
@@ -131,11 +131,11 @@ func proxyP2PConnection(ctx context.Context, node *node.Node, serviceID string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error {
|
func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error {
|
||||||
zlog.Info().Msgf("Allocating service '%s' on: %s", service, listenAddr)
|
zlog.Info("Allocating service", "service", service, "address", listenAddr)
|
||||||
// Open local port for listening
|
// Open local port for listening
|
||||||
l, err := net.Listen("tcp", listenAddr)
|
l, err := net.Listen("tcp", listenAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zlog.Error().Err(err).Msg("Error listening")
|
zlog.Error("Error listening", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
@@ -151,7 +151,7 @@ func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, serv
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return errors.New("context canceled")
|
return errors.New("context canceled")
|
||||||
default:
|
default:
|
||||||
zlog.Debug().Msg("New for connection")
|
zlog.Debug("New for connection")
|
||||||
// Listen for an incoming connection.
|
// Listen for an incoming connection.
|
||||||
conn, err := l.Accept()
|
conn, err := l.Accept()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,7 +187,7 @@ func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID stri
|
|||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
zlog.Error().Msg("Discoverer stopped")
|
zlog.Error("Discoverer stopped")
|
||||||
return
|
return
|
||||||
case tunnel := <-tunnels:
|
case tunnel := <-tunnels:
|
||||||
AddNode(servicesID, tunnel)
|
AddNode(servicesID, tunnel)
|
||||||
@@ -220,7 +220,7 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
|
|||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
zlog.Error().Msg("Discoverer stopped")
|
zlog.Error("Discoverer stopped")
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
@@ -230,14 +230,14 @@ func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID strin
|
|||||||
if logLevel == logLevelDebug {
|
if logLevel == logLevelDebug {
|
||||||
// We want to surface this debugging data only if p2p logging is set to debug
|
// We want to surface this debugging data only if p2p logging is set to debug
|
||||||
// (and not generally the whole application, as this can be really noisy)
|
// (and not generally the whole application, as this can be really noisy)
|
||||||
zlog.Debug().Any("data", ledger.LastBlock().Storage).Msg("Ledger data")
|
zlog.Debug("Ledger data", "data", ledger.LastBlock().Storage)
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range data {
|
for k, v := range data {
|
||||||
// New worker found in the ledger data as k (worker id)
|
// New worker found in the ledger data as k (worker id)
|
||||||
nd := &schema.NodeData{}
|
nd := &schema.NodeData{}
|
||||||
if err := v.Unmarshal(nd); err != nil {
|
if err := v.Unmarshal(nd); err != nil {
|
||||||
zlog.Error().Msg("cannot unmarshal node data")
|
zlog.Error("cannot unmarshal node data")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
ensureService(ctx, n, nd, k, allocate)
|
ensureService(ctx, n, nd, k, allocate)
|
||||||
@@ -278,7 +278,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
|
|||||||
// Start the service
|
// Start the service
|
||||||
port, err := freeport.GetFreePort()
|
port, err := freeport.GetFreePort()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
zlog.Error().Err(err).Msgf("Could not allocate a free port for %s", nd.ID)
|
zlog.Error("Could not allocate a free port", "error", err, "node", nd.ID)
|
||||||
cancel()
|
cancel()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -286,7 +286,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
|
|||||||
tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
|
tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
|
||||||
nd.TunnelAddress = tunnelAddress
|
nd.TunnelAddress = tunnelAddress
|
||||||
go allocateLocalService(newCtxm, n, tunnelAddress, sserv)
|
go allocateLocalService(newCtxm, n, tunnelAddress, sserv)
|
||||||
zlog.Debug().Msgf("Starting service %s on %s", sserv, tunnelAddress)
|
zlog.Debug("Starting service", "service", sserv, "address", tunnelAddress)
|
||||||
}
|
}
|
||||||
service[nd.Name] = nodeServiceData{
|
service[nd.Name] = nodeServiceData{
|
||||||
NodeData: *nd,
|
NodeData: *nd,
|
||||||
@@ -298,7 +298,7 @@ func ensureService(ctx context.Context, n *node.Node, nd *schema.NodeData, sserv
|
|||||||
if !nd.IsOnline() && !ndService.NodeData.IsOnline() {
|
if !nd.IsOnline() && !ndService.NodeData.IsOnline() {
|
||||||
ndService.CancelFunc()
|
ndService.CancelFunc()
|
||||||
delete(service, nd.Name)
|
delete(service, nd.Name)
|
||||||
zlog.Info().Msgf("Node %s is offline, deleting", nd.ID)
|
zlog.Info("Node is offline, deleting", "node", nd.ID)
|
||||||
} else if nd.IsOnline() {
|
} else if nd.IsOnline() {
|
||||||
// update last seen inside service
|
// update last seen inside service
|
||||||
nd.TunnelAddress = ndService.NodeData.TunnelAddress
|
nd.TunnelAddress = ndService.NodeData.TunnelAddress
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package schema
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
)
|
)
|
||||||
@@ -72,7 +72,7 @@ func (messages Messages) ToProto() []*proto.Message {
|
|||||||
if len(message.ToolCalls) > 0 {
|
if len(message.ToolCalls) > 0 {
|
||||||
toolCallsJSON, err := json.Marshal(message.ToolCalls)
|
toolCallsJSON, err := json.Marshal(message.ToolCalls)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Msg("failed to marshal tool_calls to JSON")
|
xlog.Warn("failed to marshal tool_calls to JSON", "error", err)
|
||||||
} else {
|
} else {
|
||||||
protoMessages[i].ToolCalls = string(toolCallsJSON)
|
protoMessages[i].ToolCalls = string(toolCallsJSON)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/xsync"
|
"github.com/mudler/LocalAI/pkg/xsync"
|
||||||
"github.com/mudler/cogito"
|
"github.com/mudler/cogito"
|
||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// AgentJobService manages agent tasks and job execution
|
// AgentJobService manages agent tasks and job execution
|
||||||
@@ -126,7 +126,7 @@ func (s *AgentJobService) LoadTasksFromFile() error {
|
|||||||
defer s.fileMutex.Unlock()
|
defer s.fileMutex.Unlock()
|
||||||
|
|
||||||
if _, err := os.Stat(s.tasksFile); os.IsNotExist(err) {
|
if _, err := os.Stat(s.tasksFile); os.IsNotExist(err) {
|
||||||
log.Debug().Msg("agent_tasks.json not found, starting with empty tasks")
|
xlog.Debug("agent_tasks.json not found, starting with empty tasks")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,12 +145,12 @@ func (s *AgentJobService) LoadTasksFromFile() error {
|
|||||||
// Schedule cron if enabled and has cron expression
|
// Schedule cron if enabled and has cron expression
|
||||||
if task.Enabled && task.Cron != "" {
|
if task.Enabled && task.Cron != "" {
|
||||||
if err := s.ScheduleCronTask(task); err != nil {
|
if err := s.ScheduleCronTask(task); err != nil {
|
||||||
log.Warn().Err(err).Str("task_id", task.ID).Msg("Failed to schedule cron task on load")
|
xlog.Warn("Failed to schedule cron task on load", "error", err, "task_id", task.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Int("count", len(tasksFile.Tasks)).Msg("Loaded tasks from file")
|
xlog.Info("Loaded tasks from file", "count", len(tasksFile.Tasks))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -190,7 +190,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
|
|||||||
defer s.fileMutex.Unlock()
|
defer s.fileMutex.Unlock()
|
||||||
|
|
||||||
if _, err := os.Stat(s.jobsFile); os.IsNotExist(err) {
|
if _, err := os.Stat(s.jobsFile); os.IsNotExist(err) {
|
||||||
log.Debug().Msg("agent_jobs.json not found, starting with empty jobs")
|
xlog.Debug("agent_jobs.json not found, starting with empty jobs")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,7 +209,7 @@ func (s *AgentJobService) LoadJobsFromFile() error {
|
|||||||
s.jobs.Set(job.ID, job)
|
s.jobs.Set(job.ID, job)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Int("count", len(jobsFile.Jobs)).Msg("Loaded jobs from file")
|
xlog.Info("Loaded jobs from file", "count", len(jobsFile.Jobs))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -267,14 +267,14 @@ func (s *AgentJobService) CreateTask(task schema.Task) (string, error) {
|
|||||||
// Schedule cron if enabled and has cron expression
|
// Schedule cron if enabled and has cron expression
|
||||||
if task.Enabled && task.Cron != "" {
|
if task.Enabled && task.Cron != "" {
|
||||||
if err := s.ScheduleCronTask(task); err != nil {
|
if err := s.ScheduleCronTask(task); err != nil {
|
||||||
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task")
|
xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
|
||||||
// Don't fail task creation if cron scheduling fails
|
// Don't fail task creation if cron scheduling fails
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save to file
|
// Save to file
|
||||||
if err := s.SaveTasksToFile(); err != nil {
|
if err := s.SaveTasksToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save tasks to file")
|
xlog.Error("Failed to save tasks to file", "error", err)
|
||||||
// Don't fail task creation if file save fails
|
// Don't fail task creation if file save fails
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,13 +304,13 @@ func (s *AgentJobService) UpdateTask(id string, task schema.Task) error {
|
|||||||
// Schedule new cron if enabled and has cron expression
|
// Schedule new cron if enabled and has cron expression
|
||||||
if task.Enabled && task.Cron != "" {
|
if task.Enabled && task.Cron != "" {
|
||||||
if err := s.ScheduleCronTask(task); err != nil {
|
if err := s.ScheduleCronTask(task); err != nil {
|
||||||
log.Warn().Err(err).Str("task_id", id).Msg("Failed to schedule cron task")
|
xlog.Warn("Failed to schedule cron task", "error", err, "task_id", id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save to file
|
// Save to file
|
||||||
if err := s.SaveTasksToFile(); err != nil {
|
if err := s.SaveTasksToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save tasks to file")
|
xlog.Error("Failed to save tasks to file", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -330,7 +330,7 @@ func (s *AgentJobService) DeleteTask(id string) error {
|
|||||||
|
|
||||||
// Save to file
|
// Save to file
|
||||||
if err := s.SaveTasksToFile(); err != nil {
|
if err := s.SaveTasksToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save tasks to file")
|
xlog.Error("Failed to save tasks to file", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -409,7 +409,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
|
|||||||
// Fetch content from URL with custom headers
|
// Fetch content from URL with custom headers
|
||||||
dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type)
|
dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("url", source.URL).Str("type", source.Type).Msg("Failed to fetch multimedia from task source")
|
xlog.Warn("Failed to fetch multimedia from task source", "error", err, "url", source.URL, "type", source.Type)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -449,7 +449,7 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
|
|||||||
// Save to file (async, don't block)
|
// Save to file (async, don't block)
|
||||||
go func() {
|
go func() {
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file")
|
xlog.Error("Failed to save jobs to file", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -544,7 +544,7 @@ func (s *AgentJobService) CancelJob(id string) error {
|
|||||||
// Save to file (async)
|
// Save to file (async)
|
||||||
go func() {
|
go func() {
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file")
|
xlog.Error("Failed to save jobs to file", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -561,7 +561,7 @@ func (s *AgentJobService) DeleteJob(id string) error {
|
|||||||
|
|
||||||
// Save to file
|
// Save to file
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file")
|
xlog.Error("Failed to save jobs to file", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -750,7 +750,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
if len(job.Images) > 0 {
|
if len(job.Images) > 0 {
|
||||||
images, err := s.convertToMultimediaContent(job.Images, JobImageType)
|
images, err := s.convertToMultimediaContent(job.Images, JobImageType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert images")
|
xlog.Warn("Failed to convert images", "error", err, "job_id", job.ID)
|
||||||
} else {
|
} else {
|
||||||
multimediaItems = append(multimediaItems, images...)
|
multimediaItems = append(multimediaItems, images...)
|
||||||
}
|
}
|
||||||
@@ -760,7 +760,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
if len(job.Videos) > 0 {
|
if len(job.Videos) > 0 {
|
||||||
videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType)
|
videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert videos")
|
xlog.Warn("Failed to convert videos", "error", err, "job_id", job.ID)
|
||||||
} else {
|
} else {
|
||||||
multimediaItems = append(multimediaItems, videos...)
|
multimediaItems = append(multimediaItems, videos...)
|
||||||
}
|
}
|
||||||
@@ -770,7 +770,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
if len(job.Audios) > 0 {
|
if len(job.Audios) > 0 {
|
||||||
audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType)
|
audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert audios")
|
xlog.Warn("Failed to convert audios", "error", err, "job_id", job.ID)
|
||||||
} else {
|
} else {
|
||||||
multimediaItems = append(multimediaItems, audios...)
|
multimediaItems = append(multimediaItems, audios...)
|
||||||
}
|
}
|
||||||
@@ -780,7 +780,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
if len(job.Files) > 0 {
|
if len(job.Files) > 0 {
|
||||||
files, err := s.convertToMultimediaContent(job.Files, JobFileType)
|
files, err := s.convertToMultimediaContent(job.Files, JobFileType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert files")
|
xlog.Warn("Failed to convert files", "error", err, "job_id", job.ID)
|
||||||
} else {
|
} else {
|
||||||
multimediaItems = append(multimediaItems, files...)
|
multimediaItems = append(multimediaItems, files...)
|
||||||
}
|
}
|
||||||
@@ -817,7 +817,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
cogito.WithContext(ctx),
|
cogito.WithContext(ctx),
|
||||||
cogito.WithMCPs(sessions...),
|
cogito.WithMCPs(sessions...),
|
||||||
cogito.WithStatusCallback(func(status string) {
|
cogito.WithStatusCallback(func(status string) {
|
||||||
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Status: %s", status)
|
xlog.Debug("Status", "job_id", job.ID, "model", modelConfig.Name, "status", status)
|
||||||
// Store trace
|
// Store trace
|
||||||
trace := schema.JobTrace{
|
trace := schema.JobTrace{
|
||||||
Type: "status",
|
Type: "status",
|
||||||
@@ -828,7 +828,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
s.jobs.Set(job.ID, job)
|
s.jobs.Set(job.ID, job)
|
||||||
}),
|
}),
|
||||||
cogito.WithReasoningCallback(func(reasoning string) {
|
cogito.WithReasoningCallback(func(reasoning string) {
|
||||||
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).Msgf("Reasoning: %s", reasoning)
|
xlog.Debug("Reasoning", "job_id", job.ID, "model", modelConfig.Name, "reasoning", reasoning)
|
||||||
// Store trace
|
// Store trace
|
||||||
trace := schema.JobTrace{
|
trace := schema.JobTrace{
|
||||||
Type: "reasoning",
|
Type: "reasoning",
|
||||||
@@ -839,9 +839,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
s.jobs.Set(job.ID, job)
|
s.jobs.Set(job.ID, job)
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
cogito.WithToolCallBack(func(t *cogito.ToolChoice, state *cogito.SessionState) cogito.ToolCallDecision {
|
||||||
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).
|
xlog.Debug("Tool call", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "reasoning", t.Reasoning, "arguments", t.Arguments)
|
||||||
Str("tool", t.Name).Str("reasoning", t.Reasoning).Interface("arguments", t.Arguments).
|
|
||||||
Msg("Tool call")
|
|
||||||
// Store trace
|
// Store trace
|
||||||
arguments := make(map[string]interface{})
|
arguments := make(map[string]interface{})
|
||||||
if t.Arguments != nil {
|
if t.Arguments != nil {
|
||||||
@@ -861,9 +859,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) {
|
||||||
log.Debug().Str("job_id", job.ID).Str("model", modelConfig.Name).
|
xlog.Debug("Tool call result", "job_id", job.ID, "model", modelConfig.Name, "tool", t.Name, "result", t.Result, "tool_arguments", t.ToolArguments)
|
||||||
Str("tool", t.Name).Str("result", t.Result).Interface("tool_arguments", t.ToolArguments).
|
|
||||||
Msg("Tool call result")
|
|
||||||
// Store trace
|
// Store trace
|
||||||
arguments := make(map[string]interface{})
|
arguments := make(map[string]interface{})
|
||||||
// Convert ToolArguments to map via JSON marshaling
|
// Convert ToolArguments to map via JSON marshaling
|
||||||
@@ -988,7 +984,7 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
|||||||
// Save to file (async)
|
// Save to file (async)
|
||||||
go func() {
|
go func() {
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file")
|
xlog.Error("Failed to save jobs to file", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -1023,7 +1019,7 @@ func (s *AgentJobService) worker(ctx context.Context) {
|
|||||||
// Execute job
|
// Execute job
|
||||||
err := s.executeJobInternal(exec.Job, exec.Task, exec.Ctx)
|
err := s.executeJobInternal(exec.Job, exec.Task, exec.Ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("job_id", exec.Job.ID).Msg("Job execution failed")
|
xlog.Error("Job execution failed", "error", err, "job_id", exec.Job.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up cancellation
|
// Clean up cancellation
|
||||||
@@ -1051,7 +1047,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
|
|||||||
// Multimedia will be fetched from task sources in ExecuteJob
|
// Multimedia will be fetched from task sources in ExecuteJob
|
||||||
_, err := s.ExecuteJob(task.ID, cronParams, "cron", nil)
|
_, err := s.ExecuteJob(task.ID, cronParams, "cron", nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("task_id", task.ID).Msg("Failed to execute cron job")
|
xlog.Error("Failed to execute cron job", "error", err, "task_id", task.ID)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1059,7 +1055,7 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.cronEntries.Set(task.ID, entryID)
|
s.cronEntries.Set(task.ID, entryID)
|
||||||
log.Info().Str("task_id", task.ID).Str("cron", cronExpr).Msg("Scheduled cron task")
|
xlog.Info("Scheduled cron task", "task_id", task.ID, "cron", cronExpr)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1069,7 +1065,7 @@ func (s *AgentJobService) UnscheduleCronTask(taskID string) {
|
|||||||
entryID := s.cronEntries.Get(taskID)
|
entryID := s.cronEntries.Get(taskID)
|
||||||
s.cronScheduler.Remove(entryID)
|
s.cronScheduler.Remove(entryID)
|
||||||
s.cronEntries.Delete(taskID)
|
s.cronEntries.Delete(taskID)
|
||||||
log.Info().Str("task_id", taskID).Msg("Unscheduled cron task")
|
xlog.Info("Unscheduled cron task", "task_id", taskID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1082,7 +1078,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
|
|||||||
return // No webhooks configured
|
return // No webhooks configured
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("job_id", job.ID).Int("webhook_count", len(webhookConfigs)).Msg("Sending webhooks")
|
xlog.Info("Sending webhooks", "job_id", job.ID, "webhook_count", len(webhookConfigs))
|
||||||
|
|
||||||
// Send all webhooks concurrently and track results
|
// Send all webhooks concurrently and track results
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -1140,7 +1136,7 @@ func (s *AgentJobService) sendWebhooks(job schema.Job, task schema.Task) {
|
|||||||
// Save to file (async)
|
// Save to file (async)
|
||||||
go func() {
|
go func() {
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file")
|
xlog.Error("Failed to save jobs to file", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -1157,11 +1153,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
|
|||||||
// Build payload
|
// Build payload
|
||||||
payload, err := s.buildWebhookPayload(job, task, webhookConfig)
|
payload, err := s.buildWebhookPayload(job, task, webhookConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to build webhook payload")
|
xlog.Error("Failed to build webhook payload", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
|
||||||
return fmt.Errorf("failed to build payload: %w", err)
|
return fmt.Errorf("failed to build payload: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Str("payload", string(payload)).Msg("Sending webhook")
|
xlog.Debug("Sending webhook", "job_id", job.ID, "webhook_url", webhookConfig.URL, "payload", string(payload))
|
||||||
|
|
||||||
// Determine HTTP method (default to POST)
|
// Determine HTTP method (default to POST)
|
||||||
method := webhookConfig.Method
|
method := webhookConfig.Method
|
||||||
@@ -1172,7 +1168,7 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
|
|||||||
// Create HTTP request
|
// Create HTTP request
|
||||||
req, err := http.NewRequest(method, webhookConfig.URL, bytes.NewBuffer(payload))
|
req, err := http.NewRequest(method, webhookConfig.URL, bytes.NewBuffer(payload))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Failed to create webhook request")
|
xlog.Error("Failed to create webhook request", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
|
||||||
return fmt.Errorf("failed to create request: %w", err)
|
return fmt.Errorf("failed to create request: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1186,11 +1182,11 @@ func (s *AgentJobService) sendWebhook(job schema.Job, task schema.Task, webhookC
|
|||||||
client := &http.Client{Timeout: 30 * time.Second}
|
client := &http.Client{Timeout: 30 * time.Second}
|
||||||
err = s.executeWithRetry(client, req)
|
err = s.executeWithRetry(client, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivery failed")
|
xlog.Error("Webhook delivery failed", "error", err, "job_id", job.ID, "webhook_url", webhookConfig.URL)
|
||||||
return fmt.Errorf("webhook delivery failed: %w", err)
|
return fmt.Errorf("webhook delivery failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("job_id", job.ID).Str("webhook_url", webhookConfig.URL).Msg("Webhook delivered successfully")
|
xlog.Info("Webhook delivered successfully", "job_id", job.ID, "webhook_url", webhookConfig.URL)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1303,10 +1299,10 @@ func (s *AgentJobService) CleanupOldJobs() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if removed > 0 {
|
if removed > 0 {
|
||||||
log.Info().Int("removed", removed).Int("retention_days", s.retentionDays).Msg("Cleaned up old jobs")
|
xlog.Info("Cleaned up old jobs", "removed", removed, "retention_days", s.retentionDays)
|
||||||
// Save to file
|
// Save to file
|
||||||
if err := s.SaveJobsToFile(); err != nil {
|
if err := s.SaveJobsToFile(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to save jobs to file after cleanup")
|
xlog.Error("Failed to save jobs to file after cleanup", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1327,10 +1323,10 @@ func (s *AgentJobService) Start(ctx context.Context) error {
|
|||||||
|
|
||||||
// Load tasks and jobs from files
|
// Load tasks and jobs from files
|
||||||
if err := s.LoadTasksFromFile(); err != nil {
|
if err := s.LoadTasksFromFile(); err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to load tasks from file")
|
xlog.Warn("Failed to load tasks from file", "error", err)
|
||||||
}
|
}
|
||||||
if err := s.LoadJobsFromFile(); err != nil {
|
if err := s.LoadJobsFromFile(); err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to load jobs from file")
|
xlog.Warn("Failed to load jobs from file", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start cron scheduler
|
// Start cron scheduler
|
||||||
@@ -1345,19 +1341,19 @@ func (s *AgentJobService) Start(ctx context.Context) error {
|
|||||||
// Schedule daily cleanup at midnight
|
// Schedule daily cleanup at midnight
|
||||||
_, err := s.cronScheduler.AddFunc("0 0 * * *", func() {
|
_, err := s.cronScheduler.AddFunc("0 0 * * *", func() {
|
||||||
if err := s.CleanupOldJobs(); err != nil {
|
if err := s.CleanupOldJobs(); err != nil {
|
||||||
log.Error().Err(err).Msg("Failed to cleanup old jobs")
|
xlog.Error("Failed to cleanup old jobs", "error", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to schedule daily cleanup")
|
xlog.Warn("Failed to schedule daily cleanup", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run initial cleanup
|
// Run initial cleanup
|
||||||
if err := s.CleanupOldJobs(); err != nil {
|
if err := s.CleanupOldJobs(); err != nil {
|
||||||
log.Warn().Err(err).Msg("Failed to run initial cleanup")
|
xlog.Warn("Failed to run initial cleanup", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Int("retention_days", s.retentionDays).Msg("AgentJobService started")
|
xlog.Info("AgentJobService started", "retention_days", s.retentionDays)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1370,7 +1366,7 @@ func (s *AgentJobService) Stop() error {
|
|||||||
if s.cronScheduler != nil {
|
if s.cronScheduler != nil {
|
||||||
s.cronScheduler.Stop()
|
s.cronScheduler.Stop()
|
||||||
}
|
}
|
||||||
log.Info().Msg("AgentJobService stopped")
|
xlog.Info("AgentJobService stopped")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1380,5 +1376,5 @@ func (s *AgentJobService) UpdateRetentionDays(days int) {
|
|||||||
if days == 0 {
|
if days == 0 {
|
||||||
s.retentionDays = 30 // Default
|
s.retentionDays = 30 // Default
|
||||||
}
|
}
|
||||||
log.Info().Int("retention_days", s.retentionDays).Msg("Updated agent job retention days")
|
xlog.Info("Updated agent job retention days", "retention_days", s.retentionDays)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
"github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
|
|
||||||
gopsutil "github.com/shirou/gopsutil/v3/process"
|
gopsutil "github.com/shirou/gopsutil/v3/process"
|
||||||
)
|
)
|
||||||
@@ -46,7 +46,7 @@ func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*sche
|
|||||||
pid, err := bms.modelLoader.GetGRPCPID(backend)
|
pid, err := bms.modelLoader.GetGRPCPID(backend)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Msg("failed to find GRPC pid")
|
xlog.Error("failed to find GRPC pid", "error", err, "model", model)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,26 +54,26 @@ func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*sche
|
|||||||
backendProcess, err := gopsutil.NewProcess(int32(pid))
|
backendProcess, err := gopsutil.NewProcess(int32(pid))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting process info")
|
xlog.Error("error getting process info", "error", err, "model", model, "pid", pid)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
memInfo, err := backendProcess.MemoryInfo()
|
memInfo, err := backendProcess.MemoryInfo()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory info")
|
xlog.Error("error getting memory info", "error", err, "model", model, "pid", pid)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
memPercent, err := backendProcess.MemoryPercent()
|
memPercent, err := backendProcess.MemoryPercent()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting memory percent")
|
xlog.Error("error getting memory percent", "error", err, "model", model, "pid", pid)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cpuPercent, err := backendProcess.CPUPercent()
|
cpuPercent, err := backendProcess.CPUPercent()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Int("pid", pid).Msg("error getting cpu percent")
|
xlog.Error("error getting cpu percent", "error", err, "model", model, "pid", pid)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,7 +92,7 @@ func (bms BackendMonitorService) CheckAndSample(modelName string) (*proto.Status
|
|||||||
|
|
||||||
status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO())
|
status, rpcErr := modelAddr.GRPC(false, nil).Status(context.TODO())
|
||||||
if rpcErr != nil {
|
if rpcErr != nil {
|
||||||
log.Warn().Msgf("backend %s experienced an error retrieving status info: %s", modelName, rpcErr.Error())
|
xlog.Warn("backend experienced an error retrieving status info", "backend", modelName, "error", rpcErr)
|
||||||
val, slbErr := bms.SampleLocalBackendProcess(modelName)
|
val, slbErr := bms.SampleLocalBackendProcess(modelName)
|
||||||
if slbErr != nil {
|
if slbErr != nil {
|
||||||
return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error())
|
return nil, fmt.Errorf("backend %s experienced an error retrieving status info via rpc: %s, then failed local node process sample: %s", modelName, rpcErr.Error(), slbErr.Error())
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, any], systemState *system.SystemState) error {
|
func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, any], systemState *system.SystemState) error {
|
||||||
@@ -62,7 +62,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
|
|||||||
g.modelLoader.DeleteExternalBackend(op.GalleryElementName)
|
g.modelLoader.DeleteExternalBackend(op.GalleryElementName)
|
||||||
} else if op.ExternalURI != "" {
|
} else if op.ExternalURI != "" {
|
||||||
// External backend installation (OCI image, URL, or path)
|
// External backend installation (OCI image, URL, or path)
|
||||||
log.Info().Str("uri", op.ExternalURI).Str("name", op.ExternalName).Str("alias", op.ExternalAlias).Msg("Installing external backend")
|
xlog.Info("Installing external backend", "uri", op.ExternalURI, "name", op.ExternalName, "alias", op.ExternalAlias)
|
||||||
err = InstallExternalBackend(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, progressCallback, op.ExternalURI, op.ExternalName, op.ExternalAlias)
|
err = InstallExternalBackend(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, progressCallback, op.ExternalURI, op.ExternalName, op.ExternalAlias)
|
||||||
// Update GalleryElementName for status tracking if a name was derived
|
// Update GalleryElementName for status tracking if a name was derived
|
||||||
if op.ExternalName != "" {
|
if op.ExternalName != "" {
|
||||||
@@ -70,8 +70,8 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Standard gallery installation
|
// Standard gallery installation
|
||||||
log.Warn().Msgf("installing backend %s", op.GalleryElementName)
|
xlog.Warn("installing backend", "backend", op.GalleryElementName)
|
||||||
log.Debug().Msgf("backend galleries: %v", g.appConfig.BackendGalleries)
|
xlog.Debug("backend galleries", "galleries", g.appConfig.BackendGalleries)
|
||||||
err = gallery.InstallBackendFromGallery(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true)
|
err = gallery.InstallBackendFromGallery(ctx, g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -85,7 +85,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend, an
|
|||||||
})
|
})
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Error().Err(err).Msgf("error installing backend %s", op.GalleryElementName)
|
xlog.Error("error installing backend", "error", err, "backend", op.GalleryElementName)
|
||||||
if !op.Delete {
|
if !op.Delete {
|
||||||
// If we didn't install the backend, we need to make sure we don't have a leftover directory
|
// If we didn't install the backend, we need to make sure we don't have a leftover directory
|
||||||
gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName)
|
gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName)
|
||||||
@@ -114,7 +114,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
|
|||||||
if name == "" { // infer it from the path
|
if name == "" { // infer it from the path
|
||||||
name = filepath.Base(backend)
|
name = filepath.Base(backend)
|
||||||
}
|
}
|
||||||
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from path")
|
xlog.Info("Installing backend from path", "backend", backend, "name", name)
|
||||||
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
||||||
Metadata: gallery.Metadata{
|
Metadata: gallery.Metadata{
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -128,7 +128,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
|
|||||||
if name == "" {
|
if name == "" {
|
||||||
return fmt.Errorf("specifying a name is required for OCI images")
|
return fmt.Errorf("specifying a name is required for OCI images")
|
||||||
}
|
}
|
||||||
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image")
|
xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
|
||||||
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
||||||
Metadata: gallery.Metadata{
|
Metadata: gallery.Metadata{
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -150,7 +150,7 @@ func InstallExternalBackend(ctx context.Context, galleries []config.Gallery, sys
|
|||||||
name = derivedName
|
name = derivedName
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image")
|
xlog.Info("Installing backend from OCI image", "backend", backend, "name", name)
|
||||||
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
if err := gallery.InstallBackend(ctx, systemState, modelLoader, &gallery.GalleryBackend{
|
||||||
Metadata: gallery.Metadata{
|
Metadata: gallery.Metadata{
|
||||||
Name: name,
|
Name: name,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package services
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"go.opentelemetry.io/otel/attribute"
|
"go.opentelemetry.io/otel/attribute"
|
||||||
"go.opentelemetry.io/otel/exporters/prometheus"
|
"go.opentelemetry.io/otel/exporters/prometheus"
|
||||||
"go.opentelemetry.io/otel/metric"
|
"go.opentelemetry.io/otel/metric"
|
||||||
@@ -49,6 +49,6 @@ func (lams LocalAIMetricsService) Shutdown() error {
|
|||||||
//// setupOTelSDK bootstraps the OpenTelemetry pipeline.
|
//// setupOTelSDK bootstraps the OpenTelemetry pipeline.
|
||||||
//// If it does not return an error, make sure to call shutdown for proper cleanup.
|
//// If it does not return an error, make sure to call shutdown for proper cleanup.
|
||||||
|
|
||||||
log.Warn().Msgf("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?")
|
xlog.Warn("LocalAIMetricsService Shutdown called, but OTelSDK proper shutdown not yet implemented?")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -208,7 +208,7 @@ func processModelOperation(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if automaticallyInstallBackend && installedModel.Backend != "" {
|
if automaticallyInstallBackend && installedModel.Backend != "" {
|
||||||
log.Debug().Msgf("Installing backend %q", installedModel.Backend)
|
xlog.Debug("Installing backend", "backend", installedModel.Backend)
|
||||||
if err := gallery.InstallBackendFromGallery(ctx, op.BackendGalleries, systemState, modelLoader, installedModel.Backend, progressCallback, false); err != nil {
|
if err := gallery.InstallBackendFromGallery(ctx, op.BackendGalleries, systemState, modelLoader, installedModel.Backend, progressCallback, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/model"
|
"github.com/mudler/LocalAI/pkg/model"
|
||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -32,10 +32,10 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
|
|||||||
// Check if it's a model gallery, or print a warning
|
// Check if it's a model gallery, or print a warning
|
||||||
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries)
|
e, found := installModel(ctx, galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries)
|
||||||
if e != nil && found {
|
if e != nil && found {
|
||||||
log.Error().Err(err).Msgf("[startup] failed installing model '%s'", url)
|
xlog.Error("[startup] failed installing model", "error", err, "model", url)
|
||||||
err = errors.Join(err, e)
|
err = errors.Join(err, e)
|
||||||
} else if !found {
|
} else if !found {
|
||||||
log.Debug().Msgf("[startup] model not found in the gallery '%s'", url)
|
xlog.Debug("[startup] model not found in the gallery", "model", url)
|
||||||
|
|
||||||
if galleryService == nil {
|
if galleryService == nil {
|
||||||
return fmt.Errorf("cannot start autoimporter, not sure how to handle this uri")
|
return fmt.Errorf("cannot start autoimporter, not sure how to handle this uri")
|
||||||
@@ -44,7 +44,7 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
|
|||||||
// TODO: we should just use the discoverModelConfig here and default to this.
|
// TODO: we should just use the discoverModelConfig here and default to this.
|
||||||
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
|
modelConfig, discoverErr := importers.DiscoverModelConfig(url, json.RawMessage{})
|
||||||
if discoverErr != nil {
|
if discoverErr != nil {
|
||||||
log.Error().Err(discoverErr).Msgf("[startup] failed to discover model config '%s'", url)
|
xlog.Error("[startup] failed to discover model config", "error", discoverErr, "model", url)
|
||||||
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
|
err = errors.Join(discoverErr, fmt.Errorf("failed to discover model config: %w", err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -76,11 +76,11 @@ func InstallModels(ctx context.Context, galleryService *services.GalleryService,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if status.Error != nil {
|
if status.Error != nil {
|
||||||
log.Error().Err(status.Error).Msgf("[startup] failed to import model '%s' from '%s'", modelConfig.Name, url)
|
xlog.Error("[startup] failed to import model", "error", status.Error, "model", modelConfig.Name, "url", url)
|
||||||
return status.Error
|
return status.Error
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Msgf("[startup] imported model '%s' from '%s'", modelConfig.Name, url)
|
xlog.Info("[startup] imported model", "model", modelConfig.Name, "url", url)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
@@ -101,7 +101,7 @@ func installModel(ctx context.Context, galleries, backendGalleries []config.Gall
|
|||||||
downloadStatus = utils.DisplayDownloadFunction
|
downloadStatus = utils.DisplayDownloadFunction
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model")
|
xlog.Info("installing model", "model", modelName, "license", model.License)
|
||||||
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries)
|
err = gallery.InstallModelFromGallery(ctx, galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err, true
|
return err, true
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/core/schema"
|
"github.com/mudler/LocalAI/core/schema"
|
||||||
"github.com/mudler/LocalAI/pkg/functions"
|
"github.com/mudler/LocalAI/pkg/functions"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Rather than pass an interface{} to the prompt template:
|
// Rather than pass an interface{} to the prompt template:
|
||||||
@@ -133,13 +133,13 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
|
|||||||
}
|
}
|
||||||
templatedChatMessage, err := e.evaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData)
|
templatedChatMessage, err := e.evaluateTemplateForChatMessage(config.TemplateConfig.ChatMessage, chatMessageData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Interface("message", chatMessageData).Str("template", config.TemplateConfig.ChatMessage).Msg("error processing message with template, skipping")
|
xlog.Error("error processing message with template, skipping", "error", err, "message", chatMessageData, "template", config.TemplateConfig.ChatMessage)
|
||||||
} else {
|
} else {
|
||||||
if templatedChatMessage == "" {
|
if templatedChatMessage == "" {
|
||||||
log.Warn().Msgf("template \"%s\" produced blank output for %+v. Skipping!", config.TemplateConfig.ChatMessage, chatMessageData)
|
xlog.Warn("template produced blank output, skipping", "template", config.TemplateConfig.ChatMessage, "message", chatMessageData)
|
||||||
continue // TODO: This continue is here intentionally to skip over the line `mess = append(mess, content)` below, and to prevent the sprintf
|
continue // TODO: This continue is here intentionally to skip over the line `mess = append(mess, content)` below, and to prevent the sprintf
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("templated message for chat: %s", templatedChatMessage)
|
xlog.Debug("templated message for chat", "message", templatedChatMessage)
|
||||||
content = templatedChatMessage
|
content = templatedChatMessage
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -203,7 +203,7 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
|
|||||||
}
|
}
|
||||||
|
|
||||||
predInput = strings.Join(mess, joinCharacter)
|
predInput = strings.Join(mess, joinCharacter)
|
||||||
log.Debug().Msgf("Prompt (before templating): %s", predInput)
|
xlog.Debug("Prompt (before templating)", "prompt", predInput)
|
||||||
|
|
||||||
promptTemplate := ChatPromptTemplate
|
promptTemplate := ChatPromptTemplate
|
||||||
|
|
||||||
@@ -221,9 +221,9 @@ func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []sche
|
|||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
predInput = templatedInput
|
predInput = templatedInput
|
||||||
log.Debug().Msgf("Template found, input modified to: %s", predInput)
|
xlog.Debug("Template found, input modified", "input", predInput)
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Msgf("Template failed loading: %s", err.Error())
|
xlog.Debug("Template failed loading", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return predInput
|
return predInput
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -41,7 +41,7 @@ require (
|
|||||||
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
|
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
|
||||||
github.com/prometheus/client_golang v1.23.2
|
github.com/prometheus/client_golang v1.23.2
|
||||||
github.com/robfig/cron/v3 v3.0.1
|
github.com/robfig/cron/v3 v3.0.1
|
||||||
github.com/rs/zerolog v1.34.0
|
github.com/mudler/xlog v0.0.2
|
||||||
github.com/russross/blackfriday v1.6.0
|
github.com/russross/blackfriday v1.6.0
|
||||||
github.com/sashabaranov/go-openai v1.41.2
|
github.com/sashabaranov/go-openai v1.41.2
|
||||||
github.com/schollz/progressbar/v3 v3.18.0
|
github.com/schollz/progressbar/v3 v3.18.0
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -518,6 +518,8 @@ github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2 h1:+WHsL/j6EWOMUiMVI
|
|||||||
github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2/go.mod h1:EA8Ashhd56o32qN7ouPKFSRUs/Z+LrRCF4v6R2Oarm8=
|
github.com/mudler/memory v0.0.0-20251216220809-d1256471a6c2/go.mod h1:EA8Ashhd56o32qN7ouPKFSRUs/Z+LrRCF4v6R2Oarm8=
|
||||||
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025 h1:WFLP5FHInarYGXi6B/Ze204x7Xy6q/I4nCZnWEyPHK0=
|
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025 h1:WFLP5FHInarYGXi6B/Ze204x7Xy6q/I4nCZnWEyPHK0=
|
||||||
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025/go.mod h1:QuIFdRstyGJt+MTTkWY+mtD7U6xwjOR6SwKUjmLZtR4=
|
github.com/mudler/water v0.0.0-20250808092830-dd90dcf09025/go.mod h1:QuIFdRstyGJt+MTTkWY+mtD7U6xwjOR6SwKUjmLZtR4=
|
||||||
|
github.com/mudler/xlog v0.0.2 h1:w3Z3HIexjYduveotEMlHAeKjTFDi15NzZLBtfV76i1Y=
|
||||||
|
github.com/mudler/xlog v0.0.2/go.mod h1:39f5vcd05Qd6GWKM8IjyHNQ7AmOx3ZM0YfhfIGhC18U=
|
||||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||||
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
|
||||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/oci"
|
"github.com/mudler/LocalAI/pkg/oci"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/mudler/LocalAI/pkg/xio"
|
"github.com/mudler/LocalAI/pkg/xio"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -70,7 +70,7 @@ func (uri URI) ReadWithAuthorizationAndCallback(ctx context.Context, basePath st
|
|||||||
// Check if the local file is rooted in basePath
|
// Check if the local file is rooted in basePath
|
||||||
err = utils.InTrustedRoot(resolvedFile, resolvedBasePath)
|
err = utils.InTrustedRoot(resolvedFile, resolvedBasePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Str("resolvedFile", resolvedFile).Str("basePath", basePath).Msg("downloader.GetURI blocked an attempt to ready a file url outside of basePath")
|
xlog.Debug("downloader.GetURI blocked an attempt to ready a file url outside of basePath", "resolvedFile", resolvedFile, "basePath", basePath)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Read the response body
|
// Read the response body
|
||||||
@@ -245,11 +245,11 @@ func (s URI) ResolveURL() string {
|
|||||||
func removePartialFile(tmpFilePath string) error {
|
func removePartialFile(tmpFilePath string) error {
|
||||||
_, err := os.Stat(tmpFilePath)
|
_, err := os.Stat(tmpFilePath)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Debug().Msgf("Removing temporary file %s", tmpFilePath)
|
xlog.Debug("Removing temporary file", "file", tmpFilePath)
|
||||||
err = os.Remove(tmpFilePath)
|
err = os.Remove(tmpFilePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err1 := fmt.Errorf("failed to remove temporary download file %s: %v", tmpFilePath, err)
|
err1 := fmt.Errorf("failed to remove temporary download file %s: %v", tmpFilePath, err)
|
||||||
log.Warn().Msg(err1.Error())
|
xlog.Warn("failed to remove temporary download file", "error", err1)
|
||||||
return err1
|
return err1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -333,7 +333,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
|
|||||||
// Check if the file already exists
|
// Check if the file already exists
|
||||||
_, err := os.Stat(filePath)
|
_, err := os.Stat(filePath)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Debug().Str("filePath", filePath).Msg("[downloader] File already exists")
|
xlog.Debug("[downloader] File already exists", "filePath", filePath)
|
||||||
// File exists, check SHA
|
// File exists, check SHA
|
||||||
if sha != "" {
|
if sha != "" {
|
||||||
// Verify SHA
|
// Verify SHA
|
||||||
@@ -343,7 +343,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
|
|||||||
}
|
}
|
||||||
if calculatedSHA == sha {
|
if calculatedSHA == sha {
|
||||||
// SHA matches, skip downloading
|
// SHA matches, skip downloading
|
||||||
log.Debug().Msgf("File %q already exists and matches the SHA. Skipping download", filePath)
|
xlog.Debug("File already exists and matches the SHA. Skipping download", "file", filePath)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// SHA doesn't match, delete the file and download again
|
// SHA doesn't match, delete the file and download again
|
||||||
@@ -351,11 +351,11 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to remove existing file %q: %v", filePath, err)
|
return fmt.Errorf("failed to remove existing file %q: %v", filePath, err)
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("Removed %q (SHA doesn't match)", filePath)
|
xlog.Debug("Removed file (SHA doesn't match)", "file", filePath)
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// SHA is missing, skip downloading
|
// SHA is missing, skip downloading
|
||||||
log.Debug().Msgf("File %q already exists. Skipping download", filePath)
|
xlog.Debug("File already exists. Skipping download", "file", filePath)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
} else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() {
|
} else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() {
|
||||||
@@ -363,7 +363,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
|
|||||||
return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url)
|
return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Msgf("Downloading %s", url)
|
xlog.Info("Downloading", "url", url)
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -480,19 +480,19 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
|
|||||||
// Verify SHA
|
// Verify SHA
|
||||||
calculatedSHA := fmt.Sprintf("%x", progress.hash.Sum(nil))
|
calculatedSHA := fmt.Sprintf("%x", progress.hash.Sum(nil))
|
||||||
if calculatedSHA != sha {
|
if calculatedSHA != sha {
|
||||||
log.Debug().Msgf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
|
xlog.Debug("SHA mismatch for file", "file", filePath, "calculated", calculatedSHA, "metadata", sha)
|
||||||
return fmt.Errorf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
|
return fmt.Errorf("SHA mismatch for file %q ( calculated: %s != metadata: %s )", filePath, calculatedSHA, sha)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Msgf("SHA missing for %q. Skipping validation", filePath)
|
xlog.Debug("SHA missing. Skipping validation", "file", filePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Msgf("File %q downloaded and verified", filePath)
|
xlog.Info("File downloaded and verified", "file", filePath)
|
||||||
if utils.IsArchive(filePath) {
|
if utils.IsArchive(filePath) {
|
||||||
basePath := filepath.Dir(filePath)
|
basePath := filepath.Dir(filePath)
|
||||||
log.Info().Msgf("File %q is an archive, uncompressing to %s", filePath, basePath)
|
xlog.Info("File is an archive, uncompressing", "file", filePath, "basePath", basePath)
|
||||||
if err := utils.ExtractArchive(filePath, basePath); err != nil {
|
if err := utils.ExtractArchive(filePath, basePath); err != nil {
|
||||||
log.Debug().Msgf("Failed decompressing %q: %s", filePath, err.Error())
|
xlog.Debug("Failed decompressing", "file", filePath, "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package functions
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -59,11 +59,11 @@ func (f Functions) ToJSONStructure(name, args string) JSONFunctionStructure {
|
|||||||
|
|
||||||
err := json.Unmarshal(dat, &prop)
|
err := json.Unmarshal(dat, &prop)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error unmarshalling dat")
|
xlog.Error("error unmarshalling dat", "error", err)
|
||||||
}
|
}
|
||||||
err = json.Unmarshal(dat2, &defsD)
|
err = json.Unmarshal(dat2, &defsD)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error unmarshalling dat2")
|
xlog.Error("error unmarshalling dat2", "error", err)
|
||||||
}
|
}
|
||||||
if js.Defs == nil {
|
if js.Defs == nil {
|
||||||
js.Defs = defsD
|
js.Defs = defsD
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/functions/grammars"
|
"github.com/mudler/LocalAI/pkg/functions/grammars"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// @Description GrammarConfig contains configuration for grammar parsing
|
// @Description GrammarConfig contains configuration for grammar parsing
|
||||||
@@ -150,22 +150,22 @@ func (g FunctionsConfig) GrammarOptions() []func(o *grammars.GrammarOption) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func CleanupLLMResult(llmresult string, functionConfig FunctionsConfig) string {
|
func CleanupLLMResult(llmresult string, functionConfig FunctionsConfig) string {
|
||||||
log.Debug().Msgf("LLM result: %s", llmresult)
|
xlog.Debug("LLM result", "result", llmresult)
|
||||||
|
|
||||||
for _, item := range functionConfig.ReplaceLLMResult {
|
for _, item := range functionConfig.ReplaceLLMResult {
|
||||||
k, v := item.Key, item.Value
|
k, v := item.Key, item.Value
|
||||||
log.Debug().Msgf("Replacing %s with %s", k, v)
|
xlog.Debug("Replacing", "key", k, "value", v)
|
||||||
re := regexp.MustCompile(k)
|
re := regexp.MustCompile(k)
|
||||||
llmresult = re.ReplaceAllString(llmresult, v)
|
llmresult = re.ReplaceAllString(llmresult, v)
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("LLM result(processed): %s", llmresult)
|
xlog.Debug("LLM result(processed)", "result", llmresult)
|
||||||
|
|
||||||
return llmresult
|
return llmresult
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string {
|
func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string {
|
||||||
log.Debug().Msgf("ParseTextContent: %s", llmresult)
|
xlog.Debug("ParseTextContent", "result", llmresult)
|
||||||
log.Debug().Msgf("CaptureLLMResult: %s", functionConfig.CaptureLLMResult)
|
xlog.Debug("CaptureLLMResult", "config", functionConfig.CaptureLLMResult)
|
||||||
|
|
||||||
for _, r := range functionConfig.CaptureLLMResult {
|
for _, r := range functionConfig.CaptureLLMResult {
|
||||||
// We use a regex to extract the JSON object from the response
|
// We use a regex to extract the JSON object from the response
|
||||||
@@ -223,15 +223,15 @@ func ParseJSON(s string) ([]map[string]any, error) {
|
|||||||
|
|
||||||
func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults {
|
func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults {
|
||||||
|
|
||||||
log.Debug().Msgf("LLM result: %s", llmresult)
|
xlog.Debug("LLM result", "result", llmresult)
|
||||||
|
|
||||||
for _, item := range functionConfig.ReplaceFunctionResults {
|
for _, item := range functionConfig.ReplaceFunctionResults {
|
||||||
k, v := item.Key, item.Value
|
k, v := item.Key, item.Value
|
||||||
log.Debug().Msgf("Replacing %s with %s", k, v)
|
xlog.Debug("Replacing", "key", k, "value", v)
|
||||||
re := regexp.MustCompile(k)
|
re := regexp.MustCompile(k)
|
||||||
llmresult = re.ReplaceAllString(llmresult, v)
|
llmresult = re.ReplaceAllString(llmresult, v)
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("LLM result(function cleanup): %s", llmresult)
|
xlog.Debug("LLM result(function cleanup)", "result", llmresult)
|
||||||
|
|
||||||
functionNameKey := defaultFunctionNameKey
|
functionNameKey := defaultFunctionNameKey
|
||||||
functionArgumentsKey := defaultFunctionArgumentsKey
|
functionArgumentsKey := defaultFunctionArgumentsKey
|
||||||
@@ -256,10 +256,10 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
|
|||||||
ss, err := ParseJSON(s)
|
ss, err := ParseJSON(s)
|
||||||
//err := json.Unmarshal([]byte(s), &ss)
|
//err := json.Unmarshal([]byte(s), &ss)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects")
|
xlog.Debug("unable to unmarshal llm result in a single object or an array of JSON objects", "error", err, "escapedLLMResult", s)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Function return: %s %+v", s, ss)
|
xlog.Debug("Function return", "result", s, "parsed", ss)
|
||||||
|
|
||||||
for _, s := range ss {
|
for _, s := range ss {
|
||||||
// The grammar defines the function name as "function", while OpenAI returns "name"
|
// The grammar defines the function name as "function", while OpenAI returns "name"
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||||
"github.com/phayes/freeport"
|
"github.com/phayes/freeport"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -50,7 +50,7 @@ const (
|
|||||||
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
|
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
|
||||||
return func(modelID, modelName, modelFile string) (*Model, error) {
|
return func(modelID, modelName, modelFile string) (*Model, error) {
|
||||||
|
|
||||||
log.Debug().Msgf("Loading Model %s with gRPC (file: %s) (backend: %s): %+v", modelID, modelFile, backend, *o)
|
xlog.Debug("Loading Model with gRPC", "modelID", modelID, "file", modelFile, "backend", backend, "options", *o)
|
||||||
|
|
||||||
var client *Model
|
var client *Model
|
||||||
|
|
||||||
@@ -67,17 +67,17 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
|||||||
if os.Getenv(env) == "" {
|
if os.Getenv(env) == "" {
|
||||||
err := os.Setenv(env, ml.ModelPath)
|
err := os.Setenv(env, ml.ModelPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("name", env).Str("modelPath", ml.ModelPath).Msg("unable to set environment variable to modelPath")
|
xlog.Error("unable to set environment variable to modelPath", "error", err, "name", env, "modelPath", ml.ModelPath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the backend is provided as external
|
// Check if the backend is provided as external
|
||||||
if uri, ok := ml.GetAllExternalBackends(o)[backend]; ok {
|
if uri, ok := ml.GetAllExternalBackends(o)[backend]; ok {
|
||||||
log.Debug().Msgf("Loading external backend: %s", uri)
|
xlog.Debug("Loading external backend", "uri", uri)
|
||||||
// check if uri is a file or a address
|
// check if uri is a file or a address
|
||||||
if fi, err := os.Stat(uri); err == nil {
|
if fi, err := os.Stat(uri); err == nil {
|
||||||
log.Debug().Msgf("external backend is file: %+v", fi)
|
xlog.Debug("external backend is file", "file", fi)
|
||||||
serverAddress, err := getFreeAddress()
|
serverAddress, err := getFreeAddress()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
|
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
|
||||||
@@ -85,43 +85,43 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
|||||||
// Make sure the process is executable
|
// Make sure the process is executable
|
||||||
process, err := ml.startProcess(uri, modelID, serverAddress)
|
process, err := ml.startProcess(uri, modelID, serverAddress)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("path", uri).Msg("failed to launch ")
|
xlog.Error("failed to launch", "error", err, "path", uri)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("GRPC Service Started")
|
xlog.Debug("GRPC Service Started")
|
||||||
|
|
||||||
client = NewModel(modelID, serverAddress, process)
|
client = NewModel(modelID, serverAddress, process)
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Msg("external backend is a uri")
|
xlog.Debug("external backend is a uri")
|
||||||
// address
|
// address
|
||||||
client = NewModel(modelID, uri, nil)
|
client = NewModel(modelID, uri, nil)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Error().Msgf("Backend not found: %s", backend)
|
xlog.Error("Backend not found", "backend", backend)
|
||||||
return nil, fmt.Errorf("backend not found: %s", backend)
|
return nil, fmt.Errorf("backend not found: %s", backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Wait for the service to start up")
|
xlog.Debug("Wait for the service to start up")
|
||||||
log.Debug().Msgf("Options: %+v", o.gRPCOptions)
|
xlog.Debug("Options", "options", o.gRPCOptions)
|
||||||
|
|
||||||
// Wait for the service to start up
|
// Wait for the service to start up
|
||||||
ready := false
|
ready := false
|
||||||
for i := 0; i < o.grpcAttempts; i++ {
|
for i := 0; i < o.grpcAttempts; i++ {
|
||||||
alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background())
|
alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background())
|
||||||
if alive {
|
if alive {
|
||||||
log.Debug().Msgf("GRPC Service Ready")
|
xlog.Debug("GRPC Service Ready")
|
||||||
ready = true
|
ready = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil && i == o.grpcAttempts-1 {
|
if err != nil && i == o.grpcAttempts-1 {
|
||||||
log.Error().Err(err).Msg("failed starting/connecting to the gRPC service")
|
xlog.Error("failed starting/connecting to the gRPC service", "error", err)
|
||||||
}
|
}
|
||||||
time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second)
|
time.Sleep(time.Duration(o.grpcAttemptsDelay) * time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !ready {
|
if !ready {
|
||||||
log.Debug().Msgf("GRPC Service NOT ready")
|
xlog.Debug("GRPC Service NOT ready")
|
||||||
if process := client.Process(); process != nil {
|
if process := client.Process(); process != nil {
|
||||||
process.Stop()
|
process.Stop()
|
||||||
}
|
}
|
||||||
@@ -133,7 +133,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
|||||||
options.ModelFile = modelFile
|
options.ModelFile = modelFile
|
||||||
options.ModelPath = ml.ModelPath
|
options.ModelPath = ml.ModelPath
|
||||||
|
|
||||||
log.Debug().Msgf("GRPC: Loading model with options: %+v", options)
|
xlog.Debug("GRPC: Loading model with options", "options", options)
|
||||||
|
|
||||||
res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options)
|
res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, &options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -156,16 +156,16 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
|||||||
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
|
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
|
||||||
o := NewOptions(opts...)
|
o := NewOptions(opts...)
|
||||||
|
|
||||||
log.Info().Str("modelID", o.modelID).Str("backend", o.backendString).Str("o.model", o.model).Msg("BackendLoader starting")
|
xlog.Info("BackendLoader starting", "modelID", o.modelID, "backend", o.backendString, "model", o.model)
|
||||||
|
|
||||||
backend := strings.ToLower(o.backendString)
|
backend := strings.ToLower(o.backendString)
|
||||||
if realBackend, exists := Aliases[backend]; exists {
|
if realBackend, exists := Aliases[backend]; exists {
|
||||||
typeAlias, exists := TypeAlias[backend]
|
typeAlias, exists := TypeAlias[backend]
|
||||||
if exists {
|
if exists {
|
||||||
log.Debug().Msgf("'%s' is a type alias of '%s' (%s)", backend, realBackend, typeAlias)
|
xlog.Debug("alias is a type alias", "alias", backend, "realBackend", realBackend, "type", typeAlias)
|
||||||
o.gRPCOptions.Type = typeAlias
|
o.gRPCOptions.Type = typeAlias
|
||||||
} else {
|
} else {
|
||||||
log.Debug().Msgf("'%s' is an alias of '%s'", backend, realBackend)
|
xlog.Debug("alias", "alias", backend, "realBackend", realBackend)
|
||||||
}
|
}
|
||||||
|
|
||||||
backend = realBackend
|
backend = realBackend
|
||||||
@@ -174,9 +174,9 @@ func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err e
|
|||||||
model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o))
|
model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if stopErr := ml.StopGRPC(only(o.modelID));stopErr != nil {
|
if stopErr := ml.StopGRPC(only(o.modelID));stopErr != nil {
|
||||||
log.Error().Err(stopErr).Str("model", o.modelID).Msg("error stopping model")
|
xlog.Error("error stopping model", "error", stopErr, "model", o.modelID)
|
||||||
}
|
}
|
||||||
log.Error().Str("modelID", o.modelID).Err(err).Msgf("Failed to load model %s with backend %s", o.modelID, o.backendString)
|
xlog.Error("Failed to load model", "modelID", o.modelID, "error", err, "backend", o.backendString)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,7 +209,7 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
|
|||||||
// Return earlier if we have a model already loaded
|
// Return earlier if we have a model already loaded
|
||||||
// (avoid looping through all the backends)
|
// (avoid looping through all the backends)
|
||||||
if m := ml.CheckIsLoaded(o.modelID); m != nil {
|
if m := ml.CheckIsLoaded(o.modelID); m != nil {
|
||||||
log.Debug().Msgf("Model '%s' already loaded", o.modelID)
|
xlog.Debug("Model already loaded", "model", o.modelID)
|
||||||
// Update last used time for LRU tracking
|
// Update last used time for LRU tracking
|
||||||
ml.updateModelLastUsed(m)
|
ml.updateModelLastUsed(m)
|
||||||
return m.GRPC(o.parallelRequests, ml.wd), nil
|
return m.GRPC(o.parallelRequests, ml.wd), nil
|
||||||
@@ -239,30 +239,30 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(autoLoadBackends) == 0 {
|
if len(autoLoadBackends) == 0 {
|
||||||
log.Error().Msg("No backends found")
|
xlog.Error("No backends found")
|
||||||
return nil, fmt.Errorf("no backends found")
|
return nil, fmt.Errorf("no backends found")
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Loading from the following backends (in order): %+v", autoLoadBackends)
|
xlog.Debug("Loading from the following backends (in order)", "backends", autoLoadBackends)
|
||||||
|
|
||||||
log.Info().Msgf("Trying to load the model '%s' with the backend '%s'", o.modelID, autoLoadBackends)
|
xlog.Info("Trying to load the model", "modelID", o.modelID, "backends", autoLoadBackends)
|
||||||
|
|
||||||
for _, key := range autoLoadBackends {
|
for _, key := range autoLoadBackends {
|
||||||
log.Info().Msgf("[%s] Attempting to load", key)
|
xlog.Info("Attempting to load", "backend", key)
|
||||||
options := append(opts, []Option{
|
options := append(opts, []Option{
|
||||||
WithBackendString(key),
|
WithBackendString(key),
|
||||||
}...)
|
}...)
|
||||||
|
|
||||||
model, modelerr := ml.backendLoader(options...)
|
model, modelerr := ml.backendLoader(options...)
|
||||||
if modelerr == nil && model != nil {
|
if modelerr == nil && model != nil {
|
||||||
log.Info().Msgf("[%s] Loads OK", key)
|
xlog.Info("Loads OK", "backend", key)
|
||||||
return model, nil
|
return model, nil
|
||||||
} else if modelerr != nil {
|
} else if modelerr != nil {
|
||||||
err = errors.Join(err, fmt.Errorf("[%s]: %w", key, modelerr))
|
err = errors.Join(err, fmt.Errorf("[%s]: %w", key, modelerr))
|
||||||
log.Info().Msgf("[%s] Fails: %s", key, modelerr.Error())
|
xlog.Info("Fails", "backend", key, "error", modelerr.Error())
|
||||||
} else if model == nil {
|
} else if model == nil {
|
||||||
err = errors.Join(err, fmt.Errorf("backend %s returned no usable model", key))
|
err = errors.Join(err, fmt.Errorf("backend %s returned no usable model", key))
|
||||||
log.Info().Msgf("[%s] Fails: %s", key, "backend returned no usable model")
|
xlog.Info("Fails", "backend", key, "error", "backend returned no usable model")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/mudler/LocalAI/pkg/system"
|
"github.com/mudler/LocalAI/pkg/system"
|
||||||
"github.com/mudler/LocalAI/pkg/utils"
|
"github.com/mudler/LocalAI/pkg/utils"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// new idea: what if we declare a struct of these here, and use a loop to check?
|
// new idea: what if we declare a struct of these here, and use a loop to check?
|
||||||
@@ -173,7 +173,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
|
|||||||
if loadingChan, isLoading := ml.loading[modelID]; isLoading {
|
if loadingChan, isLoading := ml.loading[modelID]; isLoading {
|
||||||
ml.mu.Unlock()
|
ml.mu.Unlock()
|
||||||
// Wait for the other goroutine to finish loading
|
// Wait for the other goroutine to finish loading
|
||||||
log.Debug().Str("modelID", modelID).Msg("Waiting for model to be loaded by another request")
|
xlog.Debug("Waiting for model to be loaded by another request", "modelID", modelID)
|
||||||
<-loadingChan
|
<-loadingChan
|
||||||
// Now check if the model is loaded
|
// Now check if the model is loaded
|
||||||
ml.mu.Lock()
|
ml.mu.Lock()
|
||||||
@@ -201,7 +201,7 @@ func (ml *ModelLoader) LoadModel(modelID, modelName string, loader func(string,
|
|||||||
|
|
||||||
// Load the model (this can take a long time, no lock held)
|
// Load the model (this can take a long time, no lock held)
|
||||||
modelFile := filepath.Join(ml.ModelPath, modelName)
|
modelFile := filepath.Join(ml.ModelPath, modelName)
|
||||||
log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
|
xlog.Debug("Loading model in memory from file", "file", modelFile)
|
||||||
|
|
||||||
model, err := loader(modelID, modelName, modelFile)
|
model, err := loader(modelID, modelName, modelFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -239,28 +239,28 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Model already loaded in memory: %s", s)
|
xlog.Debug("Model already loaded in memory", "model", s)
|
||||||
client := m.GRPC(false, ml.wd)
|
client := m.GRPC(false, ml.wd)
|
||||||
|
|
||||||
log.Debug().Msgf("Checking model availability (%s)", s)
|
xlog.Debug("Checking model availability", "model", s)
|
||||||
cTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
cTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
alive, err := client.HealthCheck(cTimeout)
|
alive, err := client.HealthCheck(cTimeout)
|
||||||
if !alive {
|
if !alive {
|
||||||
log.Warn().Msgf("GRPC Model not responding: %s", err.Error())
|
xlog.Warn("GRPC Model not responding", "error", err)
|
||||||
log.Warn().Msgf("Deleting the process in order to recreate it")
|
xlog.Warn("Deleting the process in order to recreate it")
|
||||||
process := m.Process()
|
process := m.Process()
|
||||||
if process == nil {
|
if process == nil {
|
||||||
log.Error().Msgf("Process not found for '%s' and the model is not responding anymore !", s)
|
xlog.Error("Process not found and the model is not responding anymore", "model", s)
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
if !process.IsAlive() {
|
if !process.IsAlive() {
|
||||||
log.Debug().Msgf("GRPC Process is not responding: %s", s)
|
xlog.Debug("GRPC Process is not responding", "model", s)
|
||||||
// stop and delete the process, this forces to re-load the model and re-create again the service
|
// stop and delete the process, this forces to re-load the model and re-create again the service
|
||||||
err := ml.deleteProcess(s)
|
err := ml.deleteProcess(s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Str("process", s).Msg("error stopping process")
|
xlog.Error("error stopping process", "error", err, "process", s)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/hpcloud/tail"
|
"github.com/hpcloud/tail"
|
||||||
"github.com/mudler/LocalAI/pkg/signals"
|
"github.com/mudler/LocalAI/pkg/signals"
|
||||||
process "github.com/mudler/go-processmanager"
|
process "github.com/mudler/go-processmanager"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true"
|
var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true"
|
||||||
@@ -20,7 +20,7 @@ var forceBackendShutdown bool = os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "
|
|||||||
func (ml *ModelLoader) deleteProcess(s string) error {
|
func (ml *ModelLoader) deleteProcess(s string) error {
|
||||||
model, ok := ml.models[s]
|
model, ok := ml.models[s]
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Debug().Msgf("Model %s not found", s)
|
xlog.Debug("Model not found", "model", s)
|
||||||
return fmt.Errorf("model %s not found", s)
|
return fmt.Errorf("model %s not found", s)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ func (ml *ModelLoader) deleteProcess(s string) error {
|
|||||||
|
|
||||||
retries := 1
|
retries := 1
|
||||||
for model.GRPC(false, ml.wd).IsBusy() {
|
for model.GRPC(false, ml.wd).IsBusy() {
|
||||||
log.Debug().Msgf("%s busy. Waiting.", s)
|
xlog.Debug("Model busy. Waiting.", "model", s)
|
||||||
dur := time.Duration(retries*2) * time.Second
|
dur := time.Duration(retries*2) * time.Second
|
||||||
if dur > retryTimeout {
|
if dur > retryTimeout {
|
||||||
dur = retryTimeout
|
dur = retryTimeout
|
||||||
@@ -37,23 +37,23 @@ func (ml *ModelLoader) deleteProcess(s string) error {
|
|||||||
retries++
|
retries++
|
||||||
|
|
||||||
if retries > 10 && forceBackendShutdown {
|
if retries > 10 && forceBackendShutdown {
|
||||||
log.Warn().Msgf("Model %s is still busy after %d retries. Forcing shutdown.", s, retries)
|
xlog.Warn("Model is still busy after retries. Forcing shutdown.", "model", s, "retries", retries)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Deleting process %s", s)
|
xlog.Debug("Deleting process", "model", s)
|
||||||
|
|
||||||
process := model.Process()
|
process := model.Process()
|
||||||
if process == nil {
|
if process == nil {
|
||||||
log.Error().Msgf("No process for %s", s)
|
xlog.Error("No process", "model", s)
|
||||||
// Nothing to do as there is no process
|
// Nothing to do as there is no process
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := process.Stop()
|
err := process.Stop()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("(deleteProcess) error while deleting process %s", s)
|
xlog.Error("(deleteProcess) error while deleting process", "error", err, "model", s)
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
@@ -95,16 +95,16 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
|
|||||||
// Check first if it has executable permissions
|
// Check first if it has executable permissions
|
||||||
if fi, err := os.Stat(grpcProcess); err == nil {
|
if fi, err := os.Stat(grpcProcess); err == nil {
|
||||||
if fi.Mode()&0111 == 0 {
|
if fi.Mode()&0111 == 0 {
|
||||||
log.Debug().Msgf("Process %s is not executable. Making it executable.", grpcProcess)
|
xlog.Debug("Process is not executable. Making it executable.", "process", grpcProcess)
|
||||||
if err := os.Chmod(grpcProcess, 0700); err != nil {
|
if err := os.Chmod(grpcProcess, 0700); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("Loading GRPC Process: %s", grpcProcess)
|
xlog.Debug("Loading GRPC Process", "process", grpcProcess)
|
||||||
|
|
||||||
log.Debug().Msgf("GRPC Service for %s will be running at: '%s'", id, serverAddress)
|
xlog.Debug("GRPC Service will be running", "id", id, "address", serverAddress)
|
||||||
|
|
||||||
workDir, err := filepath.Abs(filepath.Dir(grpcProcess))
|
workDir, err := filepath.Abs(filepath.Dir(grpcProcess))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -128,31 +128,31 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
|
|||||||
return grpcControlProcess, err
|
return grpcControlProcess, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("GRPC Service state dir: %s", grpcControlProcess.StateDir())
|
xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir())
|
||||||
|
|
||||||
signals.RegisterGracefulTerminationHandler(func() {
|
signals.RegisterGracefulTerminationHandler(func() {
|
||||||
err := grpcControlProcess.Stop()
|
err := grpcControlProcess.Stop()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msg("error while shutting down grpc process")
|
xlog.Error("error while shutting down grpc process", "error", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true})
|
t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Could not tail stderr")
|
xlog.Debug("Could not tail stderr")
|
||||||
}
|
}
|
||||||
for line := range t.Lines {
|
for line := range t.Lines {
|
||||||
log.Debug().Msgf("GRPC(%s): stderr %s", strings.Join([]string{id, serverAddress}, "-"), line.Text)
|
xlog.Debug("GRPC stderr", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
go func() {
|
go func() {
|
||||||
t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true})
|
t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Msgf("Could not tail stdout")
|
xlog.Debug("Could not tail stdout")
|
||||||
}
|
}
|
||||||
for line := range t.Lines {
|
for line := range t.Lines {
|
||||||
log.Debug().Msgf("GRPC(%s): stdout %s", strings.Join([]string{id, serverAddress}, "-"), line.Text)
|
xlog.Debug("GRPC stdout", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
process "github.com/mudler/go-processmanager"
|
process "github.com/mudler/go-processmanager"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WatchDog tracks all the requests from GRPC clients.
|
// WatchDog tracks all the requests from GRPC clients.
|
||||||
@@ -113,7 +113,7 @@ func (wd *WatchDog) GetMemoryReclaimerSettings() (enabled bool, threshold float6
|
|||||||
func (wd *WatchDog) Shutdown() {
|
func (wd *WatchDog) Shutdown() {
|
||||||
wd.Lock()
|
wd.Lock()
|
||||||
defer wd.Unlock()
|
defer wd.Unlock()
|
||||||
log.Info().Msg("[WatchDog] Shutting down watchdog")
|
xlog.Info("[WatchDog] Shutting down watchdog")
|
||||||
wd.stop <- true
|
wd.stop <- true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -191,7 +191,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Int("current", currentCount).Int("pendingLoads", pendingLoads).Int("limit", wd.lruLimit).Int("toEvict", modelsToEvict).Msg("[WatchDog] LRU enforcement triggered")
|
xlog.Debug("[WatchDog] LRU enforcement triggered", "current", currentCount, "pendingLoads", pendingLoads, "limit", wd.lruLimit, "toEvict", modelsToEvict)
|
||||||
|
|
||||||
// Build a list of models sorted by last used time (oldest first)
|
// Build a list of models sorted by last used time (oldest first)
|
||||||
var models []modelUsageInfo
|
var models []modelUsageInfo
|
||||||
@@ -217,7 +217,7 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
|
|||||||
var modelsToShutdown []string
|
var modelsToShutdown []string
|
||||||
for i := 0; i < modelsToEvict && i < len(models); i++ {
|
for i := 0; i < modelsToEvict && i < len(models); i++ {
|
||||||
m := models[i]
|
m := models[i]
|
||||||
log.Info().Str("model", m.model).Time("lastUsed", m.lastUsed).Msg("[WatchDog] LRU evicting model")
|
xlog.Info("[WatchDog] LRU evicting model", "model", m.model, "lastUsed", m.lastUsed)
|
||||||
modelsToShutdown = append(modelsToShutdown, m.model)
|
modelsToShutdown = append(modelsToShutdown, m.model)
|
||||||
// Clean up the maps while we have the lock
|
// Clean up the maps while we have the lock
|
||||||
wd.untrack(m.address)
|
wd.untrack(m.address)
|
||||||
@@ -227,21 +227,21 @@ func (wd *WatchDog) EnforceLRULimit(pendingLoads int) int {
|
|||||||
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
||||||
for _, model := range modelsToShutdown {
|
for _, model := range modelsToShutdown {
|
||||||
if err := wd.pm.ShutdownModel(model); err != nil {
|
if err := wd.pm.ShutdownModel(model); err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Msg("[WatchDog] error shutting down model during LRU eviction")
|
xlog.Error("[WatchDog] error shutting down model during LRU eviction", "error", err, "model", model)
|
||||||
}
|
}
|
||||||
log.Debug().Str("model", model).Msg("[WatchDog] LRU eviction complete")
|
xlog.Debug("[WatchDog] LRU eviction complete", "model", model)
|
||||||
}
|
}
|
||||||
|
|
||||||
return len(modelsToShutdown)
|
return len(modelsToShutdown)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wd *WatchDog) Run() {
|
func (wd *WatchDog) Run() {
|
||||||
log.Info().Msg("[WatchDog] starting watchdog")
|
xlog.Info("[WatchDog] starting watchdog")
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-wd.stop:
|
case <-wd.stop:
|
||||||
log.Info().Msg("[WatchDog] Stopping watchdog")
|
xlog.Info("[WatchDog] Stopping watchdog")
|
||||||
return
|
return
|
||||||
case <-time.After(wd.watchdogInterval):
|
case <-time.After(wd.watchdogInterval):
|
||||||
// Check if any monitoring is enabled
|
// Check if any monitoring is enabled
|
||||||
@@ -252,7 +252,7 @@ func (wd *WatchDog) Run() {
|
|||||||
wd.Unlock()
|
wd.Unlock()
|
||||||
|
|
||||||
if !busyCheck && !idleCheck && !memoryCheck {
|
if !busyCheck && !idleCheck && !memoryCheck {
|
||||||
log.Info().Msg("[WatchDog] No checks enabled, stopping watchdog")
|
xlog.Info("[WatchDog] No checks enabled, stopping watchdog")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if busyCheck {
|
if busyCheck {
|
||||||
@@ -270,19 +270,19 @@ func (wd *WatchDog) Run() {
|
|||||||
|
|
||||||
func (wd *WatchDog) checkIdle() {
|
func (wd *WatchDog) checkIdle() {
|
||||||
wd.Lock()
|
wd.Lock()
|
||||||
log.Debug().Msg("[WatchDog] Watchdog checks for idle connections")
|
xlog.Debug("[WatchDog] Watchdog checks for idle connections")
|
||||||
|
|
||||||
// Collect models to shutdown while holding the lock
|
// Collect models to shutdown while holding the lock
|
||||||
var modelsToShutdown []string
|
var modelsToShutdown []string
|
||||||
for address, t := range wd.idleTime {
|
for address, t := range wd.idleTime {
|
||||||
log.Debug().Msgf("[WatchDog] %s: idle connection", address)
|
xlog.Debug("[WatchDog] idle connection", "address", address)
|
||||||
if time.Since(t) > wd.idletimeout {
|
if time.Since(t) > wd.idletimeout {
|
||||||
log.Warn().Msgf("[WatchDog] Address %s is idle for too long, killing it", address)
|
xlog.Warn("[WatchDog] Address is idle for too long, killing it", "address", address)
|
||||||
model, ok := wd.addressModelMap[address]
|
model, ok := wd.addressModelMap[address]
|
||||||
if ok {
|
if ok {
|
||||||
modelsToShutdown = append(modelsToShutdown, model)
|
modelsToShutdown = append(modelsToShutdown, model)
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address)
|
xlog.Warn("[WatchDog] Address unresolvable", "address", address)
|
||||||
}
|
}
|
||||||
wd.untrack(address)
|
wd.untrack(address)
|
||||||
}
|
}
|
||||||
@@ -292,28 +292,28 @@ func (wd *WatchDog) checkIdle() {
|
|||||||
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
||||||
for _, model := range modelsToShutdown {
|
for _, model := range modelsToShutdown {
|
||||||
if err := wd.pm.ShutdownModel(model); err != nil {
|
if err := wd.pm.ShutdownModel(model); err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
|
xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("[WatchDog] model shut down: %s", model)
|
xlog.Debug("[WatchDog] model shut down", "model", model)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wd *WatchDog) checkBusy() {
|
func (wd *WatchDog) checkBusy() {
|
||||||
wd.Lock()
|
wd.Lock()
|
||||||
log.Debug().Msg("[WatchDog] Watchdog checks for busy connections")
|
xlog.Debug("[WatchDog] Watchdog checks for busy connections")
|
||||||
|
|
||||||
// Collect models to shutdown while holding the lock
|
// Collect models to shutdown while holding the lock
|
||||||
var modelsToShutdown []string
|
var modelsToShutdown []string
|
||||||
for address, t := range wd.busyTime {
|
for address, t := range wd.busyTime {
|
||||||
log.Debug().Msgf("[WatchDog] %s: active connection", address)
|
xlog.Debug("[WatchDog] active connection", "address", address)
|
||||||
|
|
||||||
if time.Since(t) > wd.timeout {
|
if time.Since(t) > wd.timeout {
|
||||||
model, ok := wd.addressModelMap[address]
|
model, ok := wd.addressModelMap[address]
|
||||||
if ok {
|
if ok {
|
||||||
log.Warn().Msgf("[WatchDog] Model %s is busy for too long, killing it", model)
|
xlog.Warn("[WatchDog] Model is busy for too long, killing it", "model", model)
|
||||||
modelsToShutdown = append(modelsToShutdown, model)
|
modelsToShutdown = append(modelsToShutdown, model)
|
||||||
} else {
|
} else {
|
||||||
log.Warn().Msgf("[WatchDog] Address %s unresolvable", address)
|
xlog.Warn("[WatchDog] Address unresolvable", "address", address)
|
||||||
}
|
}
|
||||||
wd.untrack(address)
|
wd.untrack(address)
|
||||||
}
|
}
|
||||||
@@ -323,9 +323,9 @@ func (wd *WatchDog) checkBusy() {
|
|||||||
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
// Now shutdown models without holding the watchdog lock to prevent deadlock
|
||||||
for _, model := range modelsToShutdown {
|
for _, model := range modelsToShutdown {
|
||||||
if err := wd.pm.ShutdownModel(model); err != nil {
|
if err := wd.pm.ShutdownModel(model); err != nil {
|
||||||
log.Error().Err(err).Str("model", model).Msg("[watchdog] error shutting down model")
|
xlog.Error("[watchdog] error shutting down model", "error", err, "model", model)
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("[WatchDog] model shut down: %s", model)
|
xlog.Debug("[WatchDog] model shut down", "model", model)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -344,7 +344,7 @@ func (wd *WatchDog) checkMemory() {
|
|||||||
// Get current memory usage (GPU if available, otherwise RAM)
|
// Get current memory usage (GPU if available, otherwise RAM)
|
||||||
aggregate := xsysinfo.GetResourceAggregateInfo()
|
aggregate := xsysinfo.GetResourceAggregateInfo()
|
||||||
if aggregate.TotalMemory == 0 {
|
if aggregate.TotalMemory == 0 {
|
||||||
log.Debug().Msg("[WatchDog] No memory information available for memory reclaimer")
|
xlog.Debug("[WatchDog] No memory information available for memory reclaimer")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -356,20 +356,11 @@ func (wd *WatchDog) checkMemory() {
|
|||||||
memoryType = "RAM"
|
memoryType = "RAM"
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().
|
xlog.Debug("[WatchDog] Memory check", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent, "loaded_models", modelCount)
|
||||||
Str("type", memoryType).
|
|
||||||
Float64("usage_percent", aggregate.UsagePercent).
|
|
||||||
Float64("threshold_percent", thresholdPercent).
|
|
||||||
Int("loaded_models", modelCount).
|
|
||||||
Msg("[WatchDog] Memory check")
|
|
||||||
|
|
||||||
// Check if usage exceeds threshold
|
// Check if usage exceeds threshold
|
||||||
if aggregate.UsagePercent > thresholdPercent {
|
if aggregate.UsagePercent > thresholdPercent {
|
||||||
log.Warn().
|
xlog.Warn("[WatchDog] Memory usage exceeds threshold, evicting LRU backend", "type", memoryType, "usage_percent", aggregate.UsagePercent, "threshold_percent", thresholdPercent)
|
||||||
Str("type", memoryType).
|
|
||||||
Float64("usage_percent", aggregate.UsagePercent).
|
|
||||||
Float64("threshold_percent", thresholdPercent).
|
|
||||||
Msg("[WatchDog] Memory usage exceeds threshold, evicting LRU backend")
|
|
||||||
|
|
||||||
// Evict the least recently used model
|
// Evict the least recently used model
|
||||||
wd.evictLRUModel()
|
wd.evictLRUModel()
|
||||||
@@ -411,10 +402,7 @@ func (wd *WatchDog) evictLRUModel() {
|
|||||||
|
|
||||||
// Get the LRU model
|
// Get the LRU model
|
||||||
lruModel := models[0]
|
lruModel := models[0]
|
||||||
log.Info().
|
xlog.Info("[WatchDog] Memory reclaimer evicting LRU model", "model", lruModel.model, "lastUsed", lruModel.lastUsed)
|
||||||
Str("model", lruModel.model).
|
|
||||||
Time("lastUsed", lruModel.lastUsed).
|
|
||||||
Msg("[WatchDog] Memory reclaimer evicting LRU model")
|
|
||||||
|
|
||||||
// Untrack the model
|
// Untrack the model
|
||||||
wd.untrack(lruModel.address)
|
wd.untrack(lruModel.address)
|
||||||
@@ -422,9 +410,9 @@ func (wd *WatchDog) evictLRUModel() {
|
|||||||
|
|
||||||
// Shutdown the model
|
// Shutdown the model
|
||||||
if err := wd.pm.ShutdownModel(lruModel.model); err != nil {
|
if err := wd.pm.ShutdownModel(lruModel.model); err != nil {
|
||||||
log.Error().Err(err).Str("model", lruModel.model).Msg("[WatchDog] error shutting down model during memory reclamation")
|
xlog.Error("[WatchDog] error shutting down model during memory reclamation", "error", err, "model", lruModel.model)
|
||||||
} else {
|
} else {
|
||||||
log.Info().Str("model", lruModel.model).Msg("[WatchDog] Memory reclaimer eviction complete")
|
xlog.Info("[WatchDog] Memory reclaimer eviction complete", "model", lruModel.model)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/jaypipes/ghw/pkg/gpu"
|
"github.com/jaypipes/ghw/pkg/gpu"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -49,11 +49,11 @@ func (s *SystemState) Capability(capMap map[string]string) string {
|
|||||||
|
|
||||||
// Check if the reported capability is in the map
|
// Check if the reported capability is in the map
|
||||||
if _, exists := capMap[reportedCapability]; exists {
|
if _, exists := capMap[reportedCapability]; exists {
|
||||||
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("Using reported capability")
|
xlog.Debug("Using reported capability", "reportedCapability", reportedCapability, "capMap", capMap)
|
||||||
return reportedCapability
|
return reportedCapability
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Str("reportedCapability", reportedCapability).Any("capMap", capMap).Msg("The requested capability was not found, using default capability")
|
xlog.Debug("The requested capability was not found, using default capability", "reportedCapability", reportedCapability, "capMap", capMap)
|
||||||
// Otherwise, return the default capability (catch-all)
|
// Otherwise, return the default capability (catch-all)
|
||||||
return defaultCapability
|
return defaultCapability
|
||||||
}
|
}
|
||||||
@@ -61,7 +61,7 @@ func (s *SystemState) Capability(capMap map[string]string) string {
|
|||||||
func (s *SystemState) getSystemCapabilities() string {
|
func (s *SystemState) getSystemCapabilities() string {
|
||||||
capability := os.Getenv(capabilityEnv)
|
capability := os.Getenv(capabilityEnv)
|
||||||
if capability != "" {
|
if capability != "" {
|
||||||
log.Info().Str("capability", capability).Msgf("Using forced capability from environment variable (%s)", capabilityEnv)
|
xlog.Info("Using forced capability from environment variable", "capability", capability, "env", capabilityEnv)
|
||||||
return capability
|
return capability
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,27 +77,27 @@ func (s *SystemState) getSystemCapabilities() string {
|
|||||||
if _, err := os.Stat(capabilityRunFile); err == nil {
|
if _, err := os.Stat(capabilityRunFile); err == nil {
|
||||||
capability, err := os.ReadFile(capabilityRunFile)
|
capability, err := os.ReadFile(capabilityRunFile)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Info().Str("capabilityRunFile", capabilityRunFile).Str("capability", string(capability)).Msgf("Using forced capability run file (%s)", capabilityRunFileEnv)
|
xlog.Info("Using forced capability run file", "capabilityRunFile", capabilityRunFile, "capability", string(capability), "env", capabilityRunFileEnv)
|
||||||
return strings.Trim(strings.TrimSpace(string(capability)), "\n")
|
return strings.Trim(strings.TrimSpace(string(capability)), "\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we are on mac and arm64, we will return metal
|
// If we are on mac and arm64, we will return metal
|
||||||
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
|
if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
|
||||||
log.Info().Msgf("Using metal capability (arm64 on mac), set %s to override", capabilityEnv)
|
xlog.Info("Using metal capability (arm64 on mac)", "env", capabilityEnv)
|
||||||
return metal
|
return metal
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we are on mac and x86, we will return darwin-x86
|
// If we are on mac and x86, we will return darwin-x86
|
||||||
if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
|
if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
|
||||||
log.Info().Msgf("Using darwin-x86 capability (amd64 on mac), set %s to override", capabilityEnv)
|
xlog.Info("Using darwin-x86 capability (amd64 on mac)", "env", capabilityEnv)
|
||||||
return darwinX86
|
return darwinX86
|
||||||
}
|
}
|
||||||
|
|
||||||
// If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t
|
// If arm64 on linux and a nvidia gpu is detected, we will return nvidia-l4t
|
||||||
if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" {
|
if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" {
|
||||||
if s.GPUVendor == nvidia {
|
if s.GPUVendor == nvidia {
|
||||||
log.Info().Msgf("Using nvidia-l4t capability (arm64 on linux), set %s to override", capabilityEnv)
|
xlog.Info("Using nvidia-l4t capability (arm64 on linux)", "env", capabilityEnv)
|
||||||
if cuda13DirExists {
|
if cuda13DirExists {
|
||||||
return nvidiaL4TCuda13
|
return nvidiaL4TCuda13
|
||||||
}
|
}
|
||||||
@@ -117,14 +117,14 @@ func (s *SystemState) getSystemCapabilities() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if s.GPUVendor == "" {
|
if s.GPUVendor == "" {
|
||||||
log.Info().Msgf("Default capability (no GPU detected), set %s to override", capabilityEnv)
|
xlog.Info("Default capability (no GPU detected)", "env", capabilityEnv)
|
||||||
return defaultCapability
|
return defaultCapability
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info().Str("Capability", s.GPUVendor).Msgf("Capability automatically detected, set %s to override", capabilityEnv)
|
xlog.Info("Capability automatically detected", "capability", s.GPUVendor, "env", capabilityEnv)
|
||||||
// If vram is less than 4GB, let's default to CPU but warn the user that they can override that via env
|
// If vram is less than 4GB, let's default to CPU but warn the user that they can override that via env
|
||||||
if s.VRAM <= 4*1024*1024*1024 {
|
if s.VRAM <= 4*1024*1024*1024 {
|
||||||
log.Warn().Msgf("VRAM is less than 4GB, defaulting to CPU. Set %s to override", capabilityEnv)
|
xlog.Warn("VRAM is less than 4GB, defaulting to CPU", "env", capabilityEnv)
|
||||||
return defaultCapability
|
return defaultCapability
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package system
|
|||||||
import (
|
import (
|
||||||
"github.com/jaypipes/ghw/pkg/gpu"
|
"github.com/jaypipes/ghw/pkg/gpu"
|
||||||
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
"github.com/mudler/LocalAI/pkg/xsysinfo"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Backend struct {
|
type Backend struct {
|
||||||
@@ -51,11 +51,11 @@ func GetSystemState(opts ...SystemStateOptions) (*SystemState, error) {
|
|||||||
|
|
||||||
// Detection is best-effort here, we don't want to fail if it fails
|
// Detection is best-effort here, we don't want to fail if it fails
|
||||||
state.gpus, _ = xsysinfo.GPUs()
|
state.gpus, _ = xsysinfo.GPUs()
|
||||||
log.Debug().Any("gpus", state.gpus).Msg("GPUs")
|
xlog.Debug("GPUs", "gpus", state.gpus)
|
||||||
state.GPUVendor, _ = detectGPUVendor(state.gpus)
|
state.GPUVendor, _ = detectGPUVendor(state.gpus)
|
||||||
log.Debug().Str("gpuVendor", state.GPUVendor).Msg("GPU vendor")
|
xlog.Debug("GPU vendor", "gpuVendor", state.GPUVendor)
|
||||||
state.VRAM, _ = xsysinfo.TotalAvailableVRAM()
|
state.VRAM, _ = xsysinfo.TotalAvailableVRAM()
|
||||||
log.Debug().Any("vram", state.VRAM).Msg("Total available VRAM")
|
xlog.Debug("Total available VRAM", "vram", state.VRAM)
|
||||||
|
|
||||||
return state, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
var base64DownloadClient http.Client = http.Client{
|
var base64DownloadClient http.Client = http.Client{
|
||||||
@@ -43,7 +43,7 @@ func GetContentURIAsBase64(s string) (string, error) {
|
|||||||
|
|
||||||
// Match any data URI prefix pattern
|
// Match any data URI prefix pattern
|
||||||
if match := dataURIPattern.FindString(s); match != "" {
|
if match := dataURIPattern.FindString(s); match != "" {
|
||||||
log.Debug().Msgf("Found data URI prefix: %s", match)
|
xlog.Debug("Found data URI prefix", "prefix", match)
|
||||||
return strings.Replace(s, match, "", 1), nil
|
return strings.Replace(s, match, "", 1), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package utils
|
|||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
var lastProgress time.Time = time.Now()
|
var lastProgress time.Time = time.Now()
|
||||||
@@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
|
|||||||
}
|
}
|
||||||
|
|
||||||
if total != "" {
|
if total != "" {
|
||||||
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
|
xlog.Info("Downloading", "fileName", fileName, "current", current, "total", total, "percentage", percentage, "eta", eta)
|
||||||
} else {
|
} else {
|
||||||
log.Info().Msgf("Downloading: %s", current)
|
xlog.Info("Downloading", "current", current)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/jaypipes/ghw"
|
"github.com/jaypipes/ghw"
|
||||||
"github.com/jaypipes/ghw/pkg/gpu"
|
"github.com/jaypipes/ghw/pkg/gpu"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GPU vendor constants
|
// GPU vendor constants
|
||||||
@@ -213,7 +213,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("nvidia-smi failed")
|
xlog.Debug("nvidia-smi failed", "error", err, "stderr", stderr.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,7 +246,7 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
|
|||||||
// Unified memory device - fall back to system RAM
|
// Unified memory device - fall back to system RAM
|
||||||
sysInfo, err := GetSystemRAMInfo()
|
sysInfo, err := GetSystemRAMInfo()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Str("device", name).Msg("failed to get system RAM for unified memory device")
|
xlog.Debug("failed to get system RAM for unified memory device", "error", err, "device", name)
|
||||||
// Still add the GPU but with zero memory info
|
// Still add the GPU but with zero memory info
|
||||||
gpus = append(gpus, GPUMemoryInfo{
|
gpus = append(gpus, GPUMemoryInfo{
|
||||||
Index: idx,
|
Index: idx,
|
||||||
@@ -267,13 +267,10 @@ func getNVIDIAGPUMemory() []GPUMemoryInfo {
|
|||||||
usagePercent = float64(usedBytes) / float64(totalBytes) * 100
|
usagePercent = float64(usedBytes) / float64(totalBytes) * 100
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().
|
xlog.Debug("using system RAM for unified memory GPU", "device", name, "system_ram_bytes", totalBytes)
|
||||||
Str("device", name).
|
|
||||||
Uint64("system_ram_bytes", totalBytes).
|
|
||||||
Msg("using system RAM for unified memory GPU")
|
|
||||||
} else if isNA {
|
} else if isNA {
|
||||||
// Unknown device with N/A values - skip memory info
|
// Unknown device with N/A values - skip memory info
|
||||||
log.Debug().Str("device", name).Msg("nvidia-smi returned N/A for unknown device")
|
xlog.Debug("nvidia-smi returned N/A for unknown device", "device", name)
|
||||||
gpus = append(gpus, GPUMemoryInfo{
|
gpus = append(gpus, GPUMemoryInfo{
|
||||||
Index: idx,
|
Index: idx,
|
||||||
Name: name,
|
Name: name,
|
||||||
@@ -329,7 +326,7 @@ func getAMDGPUMemory() []GPUMemoryInfo {
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("rocm-smi failed")
|
xlog.Debug("rocm-smi failed", "error", err, "stderr", stderr.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -416,7 +413,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("xpu-smi discovery failed")
|
xlog.Debug("xpu-smi discovery failed", "error", err, "stderr", stderr.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -431,7 +428,7 @@ func getIntelXPUSMI() []GPUMemoryInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
|
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
|
||||||
log.Debug().Err(err).Msg("failed to parse xpu-smi discovery output")
|
xlog.Debug("failed to parse xpu-smi discovery output", "error", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,7 +491,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("intel_gpu_top failed")
|
xlog.Debug("intel_gpu_top failed", "error", err, "stderr", stderr.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -523,7 +520,7 @@ func getIntelGPUTop() []GPUMemoryInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := json.Unmarshal([]byte(lastJSON), &result); err != nil {
|
if err := json.Unmarshal([]byte(lastJSON), &result); err != nil {
|
||||||
log.Debug().Err(err).Msg("failed to parse intel_gpu_top output")
|
xlog.Debug("failed to parse intel_gpu_top output", "error", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -557,7 +554,7 @@ func GetResourceInfo() ResourceInfo {
|
|||||||
// No GPU - fall back to system RAM
|
// No GPU - fall back to system RAM
|
||||||
ramInfo, err := GetSystemRAMInfo()
|
ramInfo, err := GetSystemRAMInfo()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug().Err(err).Msg("failed to get system RAM info")
|
xlog.Debug("failed to get system RAM info", "error", err)
|
||||||
return ResourceInfo{
|
return ResourceInfo{
|
||||||
Type: "ram",
|
Type: "ram",
|
||||||
Available: false,
|
Available: false,
|
||||||
@@ -601,7 +598,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
log.Debug().Err(err).Str("stderr", stderr.String()).Msg("vulkaninfo failed")
|
xlog.Debug("vulkaninfo failed", "error", err, "stderr", stderr.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -620,7 +617,7 @@ func getVulkanGPUMemory() []GPUMemoryInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
|
if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
|
||||||
log.Debug().Err(err).Msg("failed to parse vulkaninfo output")
|
xlog.Debug("failed to parse vulkaninfo output", "error", err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package xsysinfo
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/mudler/memory"
|
"github.com/mudler/memory"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/mudler/xlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SystemRAMInfo contains system RAM usage information
|
// SystemRAMInfo contains system RAM usage information
|
||||||
@@ -25,7 +25,7 @@ func GetSystemRAMInfo() (*SystemRAMInfo, error) {
|
|||||||
if total > 0 {
|
if total > 0 {
|
||||||
usagePercent = float64(used) / float64(total) * 100
|
usagePercent = float64(used) / float64(total) * 100
|
||||||
}
|
}
|
||||||
log.Debug().Uint64("total", total).Uint64("used", used).Uint64("free", free).Float64("usage_percent", usagePercent).Msg("System RAM Info")
|
xlog.Debug("System RAM Info", "total", total, "used", used, "free", free, "usage_percent", usagePercent)
|
||||||
return &SystemRAMInfo{
|
return &SystemRAMInfo{
|
||||||
Total: total,
|
Total: total,
|
||||||
Used: used,
|
Used: used,
|
||||||
|
|||||||
@@ -6,12 +6,11 @@ import (
|
|||||||
|
|
||||||
. "github.com/onsi/ginkgo/v2"
|
. "github.com/onsi/ginkgo/v2"
|
||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
"github.com/rs/zerolog"
|
"github.com/mudler/xlog"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLocalAI(t *testing.T) {
|
func TestLocalAI(t *testing.T) {
|
||||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
|
||||||
RegisterFailHandler(Fail)
|
RegisterFailHandler(Fail)
|
||||||
RunSpecs(t, "LocalAI test suite")
|
RunSpecs(t, "LocalAI test suite")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,7 @@ import (
|
|||||||
|
|
||||||
. "github.com/onsi/ginkgo/v2"
|
. "github.com/onsi/ginkgo/v2"
|
||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
"github.com/rs/zerolog"
|
"github.com/mudler/xlog"
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
|
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
"github.com/mudler/LocalAI/pkg/grpc"
|
"github.com/mudler/LocalAI/pkg/grpc"
|
||||||
@@ -187,7 +186,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
|
|||||||
|
|
||||||
for i, k := range keys {
|
for i, k := range keys {
|
||||||
s := sims[i]
|
s := sims[i]
|
||||||
log.Debug().Float32("similarity", s).Msgf("key: %v", k)
|
xlog.Debug("key", "similarity", s, "key", k)
|
||||||
}
|
}
|
||||||
|
|
||||||
Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5}))
|
Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5}))
|
||||||
@@ -214,7 +213,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
|
|||||||
|
|
||||||
for i, k := range ks {
|
for i, k := range ks {
|
||||||
s := sims[i]
|
s := sims[i]
|
||||||
log.Debug().Float32("similarity", s).Msgf("key: %v", k)
|
xlog.Debug("key", "similarity", s, "key", k)
|
||||||
}
|
}
|
||||||
|
|
||||||
Expect(ks[0]).To(Equal(keys[0]))
|
Expect(ks[0]).To(Equal(keys[0]))
|
||||||
|
|||||||
Reference in New Issue
Block a user