mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-05-04 14:09:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(backends): add system backend, refactor (#6059)

Browse Source 
- Add a system backend path
- Refactor and consolidate system information in system state
- Use system state in all the components to figure out the system paths
  to used whenever needed
- Refactor BackendConfig -> ModelConfig. This was otherway misleading as
  now we do have a backend configuration which is not the model config.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-08-14 19:38:26 +02:00
committed by GitHub
parent 253b7537dc
commit 089efe05fd
85 changed files with 999 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/services/backend_monitor.go
View File

@@ -16,21 +16,21 @@ import (
)
type BackendMonitorService struct {
backendConfigLoader *config.BackendConfigLoader
modelLoader *model.ModelLoader
options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name.
modelConfigLoader *config.ModelConfigLoader
modelLoader *model.ModelLoader
options *config.ApplicationConfig // Taking options in case we need to inspect ExternalGRPCBackends, though that's out of scope for now, hence the name.
}
func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.BackendConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService {
func NewBackendMonitorService(modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, appConfig *config.ApplicationConfig) *BackendMonitorService {
return &BackendMonitorService{
modelLoader: modelLoader,
backendConfigLoader: configLoader,
options: appConfig,
modelLoader: modelLoader,
modelConfigLoader: configLoader,
options: appConfig,
}
}
func (bms BackendMonitorService) getModelLoaderIDFromModelName(modelName string) (string, error) {
config, exists := bms.backendConfigLoader.GetBackendConfig(modelName)
config, exists := bms.modelConfigLoader.GetModelConfig(modelName)
var backendId string
if exists {
backendId = config.Model
@@ -47,7 +47,7 @@ func (bms BackendMonitorService) getModelLoaderIDFromModelName(modelName string)
}
func (bms *BackendMonitorService) SampleLocalBackendProcess(model string) (*schema.BackendMonitorResponse, error) {
config, exists := bms.backendConfigLoader.GetBackendConfig(model)
config, exists := bms.modelConfigLoader.GetModelConfig(model)
var backend string
if exists {
backend = config.Model

8
core/services/backends.go
View File

@@ -20,21 +20,21 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend], s
var err error
if op.Delete {
err = gallery.DeleteBackendFromSystem(g.appConfig.BackendsPath, op.GalleryElementName)
err = gallery.DeleteBackendFromSystem(g.appConfig.SystemState, op.GalleryElementName)
g.modelLoader.DeleteExternalBackend(op.GalleryElementName)
} else {
log.Warn().Msgf("installing backend %s", op.GalleryElementName)
log.Debug().Msgf("backend galleries: %v", g.appConfig.BackendGalleries)
err = gallery.InstallBackendFromGallery(g.appConfig.BackendGalleries, systemState, op.GalleryElementName, g.appConfig.BackendsPath, progressCallback, true)
err = gallery.InstallBackendFromGallery(g.appConfig.BackendGalleries, systemState, op.GalleryElementName, progressCallback, true)
if err == nil {
err = gallery.RegisterBackends(g.appConfig.BackendsPath, g.modelLoader)
err = gallery.RegisterBackends(systemState, g.modelLoader)
}
}
if err != nil {
log.Error().Err(err).Msgf("error installing backend %s", op.GalleryElementName)
if !op.Delete {
// If we didn't install the backend, we need to make sure we don't have a leftover directory
gallery.DeleteBackendFromSystem(g.appConfig.BackendsPath, op.GalleryElementName)
gallery.DeleteBackendFromSystem(systemState, op.GalleryElementName)
}
return err
}

10
core/services/gallery.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
)
type GalleryService struct {
@@ -52,7 +51,7 @@ func (g *GalleryService) GetAllStatus() map[string]*GalleryOpStatus {
return g.statuses
}
func (g *GalleryService) Start(c context.Context, cl *config.BackendConfigLoader) error {
func (g *GalleryService) Start(c context.Context, cl *config.ModelConfigLoader, systemState *system.SystemState) error {
// updates the status with an error
var updateError func(id string, e error)
if !g.appConfig.OpaqueErrors {
@@ -65,11 +64,6 @@ func (g *GalleryService) Start(c context.Context, cl *config.BackendConfigLoader
}
}
systemState, err := system.GetSystemState()
if err != nil {
log.Error().Err(err).Msg("failed to get system state")
}
go func() {
for {
select {
@@ -82,7 +76,7 @@ func (g *GalleryService) Start(c context.Context, cl *config.BackendConfigLoader
}
case op := <-g.ModelGalleryChannel:
err := g.modelHandler(&op, cl)
err := g.modelHandler(&op, cl, systemState)
if err != nil {
updateError(op.ID, err)
}

6
core/services/list_models.go
View File

@@ -14,7 +14,7 @@ const (
ALWAYS_INCLUDE
)
func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter config.BackendConfigFilterFn, looseFilePolicy LooseFilePolicy) ([]string, error) {
func ListModels(bcl *config.ModelConfigLoader, ml *model.ModelLoader, filter config.ModelConfigFilterFn, looseFilePolicy LooseFilePolicy) ([]string, error) {
var skipMap map[string]interface{} = map[string]interface{}{}
@@ -22,7 +22,7 @@ func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter c
// Start with known configurations
for _, c := range bcl.GetBackendConfigsByFilter(filter) {
for _, c := range bcl.GetModelConfigsByFilter(filter) {
// Is this better than looseFilePolicy <= SKIP_IF_CONFIGURED ? less performant but more readable?
if (looseFilePolicy == SKIP_IF_CONFIGURED) || (looseFilePolicy == LOOSE_ONLY) {
skipMap[c.Model] = nil
@@ -50,7 +50,7 @@ func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter c
return dataModels, nil
}
func CheckIfModelExists(bcl *config.BackendConfigLoader, ml *model.ModelLoader, modelName string, looseFilePolicy LooseFilePolicy) (bool, error) {
func CheckIfModelExists(bcl *config.ModelConfigLoader, ml *model.ModelLoader, modelName string, looseFilePolicy LooseFilePolicy) (bool, error) {
filter, err := config.BuildNameFilterFn(modelName)
if err != nil {
return false, err

67
core/services/models.go
View File

@@ -3,7 +3,6 @@ package services
import (
"encoding/json"
"os"
"path/filepath"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
@@ -12,7 +11,7 @@ import (
"gopkg.in/yaml.v2"
)
func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *config.BackendConfigLoader) error {
func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *config.ModelConfigLoader, systemState *system.SystemState) error {
utils.ResetDownloadTimers()
g.UpdateStatus(op.ID, &GalleryOpStatus{Message: "processing", Progress: 0})
@@ -23,18 +22,18 @@ func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *c
utils.DisplayDownloadFunction(fileName, current, total, percentage)
}
err := processModelOperation(op, g.appConfig.ModelPath, g.appConfig.BackendsPath, g.appConfig.EnforcePredownloadScans, g.appConfig.AutoloadBackendGalleries, progressCallback)
err := processModelOperation(op, systemState, g.appConfig.EnforcePredownloadScans, g.appConfig.AutoloadBackendGalleries, progressCallback)
if err != nil {
return err
}
// Reload models
err = cl.LoadBackendConfigsFromPath(g.appConfig.ModelPath)
err = cl.LoadModelConfigsFromPath(systemState.Model.ModelsPath)
if err != nil {
return err
}
err = cl.Preload(g.appConfig.ModelPath)
err = cl.Preload(systemState.Model.ModelsPath)
if err != nil {
return err
}
@@ -50,26 +49,21 @@ func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *c
return nil
}
func installModelFromRemoteConfig(modelPath string, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery, backendBasePath string) error {
config, err := gallery.GetGalleryConfigFromURL[gallery.ModelConfig](req.URL, modelPath)
func installModelFromRemoteConfig(systemState *system.SystemState, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery) error {
config, err := gallery.GetGalleryConfigFromURL[gallery.ModelConfig](req.URL, systemState.Model.ModelsPath)
if err != nil {
return err
}
config.Files = append(config.Files, req.AdditionalFiles...)
installedModel, err := gallery.InstallModel(modelPath, req.Name, &config, req.Overrides, downloadStatus, enforceScan)
installedModel, err := gallery.InstallModel(systemState, req.Name, &config, req.Overrides, downloadStatus, enforceScan)
if err != nil {
return err
}
if automaticallyInstallBackend && installedModel.Backend != "" {
systemState, err := system.GetSystemState()
if err != nil {
return err
}
if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, backendBasePath, downloadStatus, false); err != nil {
if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, downloadStatus, false); err != nil {
return err
}
}
@@ -82,22 +76,22 @@ type galleryModel struct {
ID string `json:"id"`
}
func processRequests(modelPath, backendBasePath string, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel) error {
func processRequests(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel) error {
var err error
for _, r := range requests {
utils.ResetDownloadTimers()
if r.ID == "" {
err = installModelFromRemoteConfig(modelPath, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, backendBasePath)
err = installModelFromRemoteConfig(systemState, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries)
} else {
err = gallery.InstallModelFromGallery(
galleries, backendGalleries, r.ID, modelPath, backendBasePath, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend)
galleries, backendGalleries, systemState, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend)
}
}
return err
}
func ApplyGalleryFromFile(modelPath, backendBasePath string, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error {
func ApplyGalleryFromFile(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error {
dat, err := os.ReadFile(s)
if err != nil {
return err
@@ -108,58 +102,35 @@ func ApplyGalleryFromFile(modelPath, backendBasePath string, enforceScan, automa
return err
}
return processRequests(modelPath, backendBasePath, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests)
return processRequests(systemState, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests)
}
func ApplyGalleryFromString(modelPath, backendBasePath string, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error {
func ApplyGalleryFromString(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error {
var requests []galleryModel
err := json.Unmarshal([]byte(s), &requests)
if err != nil {
return err
}
return processRequests(modelPath, backendBasePath, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests)
return processRequests(systemState, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests)
}
// processModelOperation handles the installation or deletion of a model
func processModelOperation(
op *GalleryOp[gallery.GalleryModel],
modelPath string,
backendBasePath string,
systemState *system.SystemState,
enforcePredownloadScans bool,
automaticallyInstallBackend bool,
progressCallback func(string, string, string, float64),
) error {
// delete a model
if op.Delete {
modelConfig := &config.BackendConfig{}
// Galleryname is the name of the model in this case
dat, err := os.ReadFile(filepath.Join(modelPath, op.GalleryElementName+".yaml"))
if err != nil {
return err
}
err = yaml.Unmarshal(dat, modelConfig)
if err != nil {
return err
}
files := []string{}
// Remove the model from the config
if modelConfig.Model != "" {
files = append(files, modelConfig.ModelFileName())
}
if modelConfig.MMProj != "" {
files = append(files, modelConfig.MMProjFileName())
}
return gallery.DeleteModelFromSystem(modelPath, op.GalleryElementName, files)
return gallery.DeleteModelFromSystem(systemState, op.GalleryElementName)
}
// if the request contains a gallery name, we apply the gallery from the gallery list
if op.GalleryElementName != "" {
return gallery.InstallModelFromGallery(op.Galleries, op.BackendGalleries, op.GalleryElementName, modelPath, backendBasePath, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend)
return gallery.InstallModelFromGallery(op.Galleries, op.BackendGalleries, systemState, op.GalleryElementName, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend)
// } else if op.ConfigURL != "" {
// err := startup.InstallModels(op.Galleries, modelPath, enforcePredownloadScans, progressCallback, op.ConfigURL)
// if err != nil {
@@ -167,6 +138,6 @@ func processModelOperation(
// }
// return cl.Preload(modelPath)
} else {
return installModelFromRemoteConfig(modelPath, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend, op.BackendGalleries, backendBasePath)
return installModelFromRemoteConfig(systemState, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend, op.BackendGalleries)
}
}