mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-04-04 07:01:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(backends): add system backend, refactor (#6059)

Browse Source 
- Add a system backend path
- Refactor and consolidate system information in system state
- Use system state in all the components to figure out the system paths
  to used whenever needed
- Refactor BackendConfig -> ModelConfig. This was otherway misleading as
  now we do have a backend configuration which is not the model config.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-08-14 19:38:26 +02:00
committed by GitHub
parent 253b7537dc
commit 089efe05fd
85 changed files with 999 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/cli/worker/worker.go
View File

@@ -1,8 +1,9 @@
package worker
type WorkerFlags struct {
BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
ExtraLLamaCPPArgs string `name:"llama-cpp-args" env:"LOCALAI_EXTRA_LLAMA_CPP_ARGS,EXTRA_LLAMA_CPP_ARGS" help:"Extra arguments to pass to llama-cpp-rpc-server"`
BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/usr/share/localai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
ExtraLLamaCPPArgs string `name:"llama-cpp-args" env:"LOCALAI_EXTRA_LLAMA_CPP_ARGS,EXTRA_LLAMA_CPP_ARGS" help:"Extra arguments to pass to llama-cpp-rpc-server"`
}
type Worker struct {

17
core/cli/worker/worker_llamacpp.go
View File

@@ -10,6 +10,7 @@ import (
cliContext "github.com/mudler/LocalAI/core/cli/context"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
)
@@ -21,20 +22,19 @@ const (
llamaCPPRPCBinaryName = "llama-cpp-rpc-server"
)
func findLLamaCPPBackend(backendSystemPath string) (string, error) {
backends, err := gallery.ListSystemBackends(backendSystemPath)
func findLLamaCPPBackend(systemState *system.SystemState) (string, error) {
backends, err := gallery.ListSystemBackends(systemState)
if err != nil {
log.Warn().Msgf("Failed listing system backends: %s", err)
return "", err
}
log.Debug().Msgf("System backends: %v", backends)
backendPath := ""
backend, ok := backends.Get("llama-cpp")
if !ok {
return "", errors.New("llama-cpp backend not found, install it first")
}
backendPath = filepath.Dir(backend.RunFile)
backendPath := filepath.Dir(backend.RunFile)
if backendPath == "" {
return "", errors.New("llama-cpp backend not found, install it first")
@@ -54,7 +54,14 @@ func (r *LLamaCPP) Run(ctx *cliContext.Context) error {
return fmt.Errorf("usage: local-ai worker llama-cpp-rpc -- <llama-rpc-server-args>")
}
grpcProcess, err := findLLamaCPPBackend(r.BackendsPath)
systemState, err := system.GetSystemState(
system.WithBackendPath(r.BackendsPath),
system.WithBackendSystemPath(r.BackendsSystemPath),
)
if err != nil {
return err
}
grpcProcess, err := findLLamaCPPBackend(systemState)
if err != nil {
return err
}

11
core/cli/worker/worker_p2p.go
View File

@@ -10,6 +10,7 @@ import (
cliContext "github.com/mudler/LocalAI/core/cli/context"
"github.com/mudler/LocalAI/core/p2p"
"github.com/mudler/LocalAI/pkg/system"
"github.com/phayes/freeport"
"github.com/rs/zerolog/log"
)
@@ -25,6 +26,14 @@ type P2P struct {
func (r *P2P) Run(ctx *cliContext.Context) error {
systemState, err := system.GetSystemState(
system.WithBackendPath(r.BackendsPath),
system.WithBackendSystemPath(r.BackendsSystemPath),
)
if err != nil {
return err
}
// Check if the token is set
// as we always need it.
if r.Token == "" {
@@ -60,7 +69,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
for {
log.Info().Msgf("Starting llama-cpp-rpc-server on '%s:%d'", address, port)
grpcProcess, err := findLLamaCPPBackend(r.BackendsPath)
grpcProcess, err := findLLamaCPPBackend(systemState)
if err != nil {
log.Error().Err(err).Msg("Failed to find llama-cpp-rpc-server")
return