mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-10-21 22:04:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(backends): add system backend, refactor (#6059)

Browse Source 
- Add a system backend path
- Refactor and consolidate system information in system state
- Use system state in all the components to figure out the system paths
  to used whenever needed
- Refactor BackendConfig -> ModelConfig. This was otherway misleading as
  now we do have a backend configuration which is not the model config.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto authored and GitHub committed 2025-08-14 19:38:26 +02:00
1 parent 253b7537dc
commit 089efe05fd
85 files changed
+999 -652
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

core/backend/detection.go
+2 -2
View File
@@ -13,9 +13,9 @@ func Detection(
sourceFile string,
loader *model.ModelLoader,
appConfig *config.ApplicationConfig,
backendConfig config.BackendConfig,
modelConfig config.ModelConfig,
) (*proto.DetectResponse, error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
detectionModel, err := loader.Load(opts...)
if err != nil {
return nil, err
core/backend/embeddings.go
+3 -3
View File
@@ -9,9 +9,9 @@ import (
model "github.com/mudler/LocalAI/pkg/model"
)
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
inferenceModel, err := loader.Load(opts...)
if err != nil {
@@ -23,7 +23,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendCo
switch model := inferenceModel.(type) {
case grpc.Backend:
fn = func() ([]float32, error) {
predictOptions := gRPCPredictOpts(backendConfig, loader.ModelPath)
predictOptions := gRPCPredictOpts(modelConfig, loader.ModelPath)
if len(tokens) > 0 {
embeds := []int32{}
core/backend/image.go
+4 -4
View File
@@ -7,9 +7,9 @@ import (
model "github.com/mudler/LocalAI/pkg/model"
)
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
inferenceModel, err := loader.Load(
opts...,
)
@@ -27,12 +27,12 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
Mode: int32(mode),
Step: int32(step),
Seed: int32(seed),
CLIPSkip: int32(backendConfig.Diffusers.ClipSkip),
CLIPSkip: int32(modelConfig.Diffusers.ClipSkip),
PositivePrompt: positive_prompt,
NegativePrompt: negative_prompt,
Dst: dst,
Src: src,
EnableParameters: backendConfig.Diffusers.EnableParameters,
EnableParameters: modelConfig.Diffusers.EnableParameters,
RefImages: refImages,
})
return err
core/backend/llm.go
+3 -3
View File
@@ -35,7 +35,7 @@ type TokenUsage struct {
TimingTokenGeneration float64
}
func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c *config.BackendConfig, cl *config.BackendConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c *config.ModelConfig, cl *config.ModelConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
modelFile := c.Model
// Check if the modelFile exists, if it doesn't try to load it from the gallery
@@ -47,7 +47,7 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
if !slices.Contains(modelNames, c.Name) {
utils.ResetDownloadTimers()
// if we failed to load the model, we try to download it
err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, c.Name, loader.ModelPath, o.BackendsPath, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, o.SystemState, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
if err != nil {
log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile)
//return nil, err
@@ -201,7 +201,7 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
var cutstrings map[string]*regexp.Regexp = make(map[string]*regexp.Regexp)
var mu sync.Mutex = sync.Mutex{}
func Finetune(config config.BackendConfig, input, prediction string) string {
func Finetune(config config.ModelConfig, input, prediction string) string {
if config.Echo {
prediction = input + prediction
}
core/backend/llm_test.go
+2 -2
View File
@@ -12,14 +12,14 @@ import (
var _ = Describe("LLM tests", func() {
Context("Finetune LLM output", func() {
var (
testConfig config.BackendConfig
testConfig config.ModelConfig
input string
prediction string
result string
)
BeforeEach(func() {
testConfig = config.BackendConfig{
testConfig = config.ModelConfig{
PredictionOptions: schema.PredictionOptions{
Echo: false,
},
core/backend/options.go
+4 -4
View File
@@ -11,7 +11,7 @@ import (
"github.com/rs/zerolog/log"
)
func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
name := c.Name
if name == "" {
name = c.Model
@@ -58,7 +58,7 @@ func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...
return append(defOpts, opts...)
}
func getSeed(c config.BackendConfig) int32 {
func getSeed(c config.ModelConfig) int32 {
var seed int32 = config.RAND_SEED
if c.Seed != nil {
@@ -72,7 +72,7 @@ func getSeed(c config.BackendConfig) int32 {
return seed
}
func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
func grpcModelOpts(c config.ModelConfig) *pb.ModelOptions {
b := 512
if c.Batch != 0 {
b = c.Batch
@@ -195,7 +195,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
}
}
func gRPCPredictOpts(c config.BackendConfig, modelPath string) *pb.PredictOptions {
func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions {
promptCachePath := ""
if c.PromptCachePath != "" {
p := filepath.Join(modelPath, c.PromptCachePath)
core/backend/rerank.go
+2 -2
View File
@@ -9,8 +9,8 @@ import (
model "github.com/mudler/LocalAI/pkg/model"
)
func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error) {
opts := ModelOptions(backendConfig, appConfig)
func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, modelConfig config.ModelConfig) (*proto.RerankResult, error) {
opts := ModelOptions(modelConfig, appConfig)
rerankModel, err := loader.Load(opts...)
if err != nil {
return nil, err
core/backend/soundgeneration.go
+3 -3
View File
@@ -21,10 +21,10 @@ func SoundGeneration(
sourceDivisor *int32,
loader *model.ModelLoader,
appConfig *config.ApplicationConfig,
backendConfig config.BackendConfig,
modelConfig config.ModelConfig,
) (string, *proto.Result, error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
soundGenModel, err := loader.Load(opts...)
if err != nil {
return "", nil, err
@@ -49,7 +49,7 @@ func SoundGeneration(
res, err := soundGenModel.SoundGeneration(context.Background(), &proto.SoundGenerationRequest{
Text: text,
Model: backendConfig.Model,
Model: modelConfig.Model,
Dst: filePath,
Sample: doSample,
Duration: duration,
core/backend/token_metrics.go
+2 -2
View File
@@ -13,9 +13,9 @@ func TokenMetrics(
modelFile string,
loader *model.ModelLoader,
appConfig *config.ApplicationConfig,
backendConfig config.BackendConfig) (*proto.MetricsResponse, error) {
modelConfig config.ModelConfig) (*proto.MetricsResponse, error) {
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
opts := ModelOptions(modelConfig, appConfig, model.WithModel(modelFile))
model, err := loader.Load(opts...)
if err != nil {
return nil, err
core/backend/tokenize.go
+3 -3
View File
@@ -7,19 +7,19 @@ import (
"github.com/mudler/LocalAI/pkg/model"
)
func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (schema.TokenizeResponse, error) {
func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (schema.TokenizeResponse, error) {
var inferenceModel grpc.Backend
var err error
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
inferenceModel, err = loader.Load(opts...)
if err != nil {
return schema.TokenizeResponse{}, err
}
defer loader.Close()
predictOptions := gRPCPredictOpts(backendConfig, loader.ModelPath)
predictOptions := gRPCPredictOpts(modelConfig, loader.ModelPath)
predictOptions.Prompt = s
// tokenize the string
core/backend/transcript.go
+5 -5
View File
@@ -12,13 +12,13 @@ import (
"github.com/mudler/LocalAI/pkg/model"
)
func ModelTranscription(audio, language string, translate bool, ml *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (*schema.TranscriptionResult, error) {
func ModelTranscription(audio, language string, translate bool, ml *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (*schema.TranscriptionResult, error) {
if backendConfig.Backend == "" {
backendConfig.Backend = model.WhisperBackend
if modelConfig.Backend == "" {
modelConfig.Backend = model.WhisperBackend
}
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
transcriptionModel, err := ml.Load(opts...)
if err != nil {
@@ -34,7 +34,7 @@ func ModelTranscription(audio, language string, translate bool, ml *model.ModelL
Dst: audio,
Language: language,
Translate: translate,
Threads: uint32(*backendConfig.Threads),
Threads: uint32(*modelConfig.Threads),
})
if err != nil {
return nil, err
core/backend/tts.go
+6 -6
View File
@@ -19,9 +19,9 @@ func ModelTTS(
language string,
loader *model.ModelLoader,
appConfig *config.ApplicationConfig,
backendConfig config.BackendConfig,
modelConfig config.ModelConfig,
) (string, *proto.Result, error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
ttsModel, err := loader.Load(opts...)
if err != nil {
return "", nil, err
@@ -29,7 +29,7 @@ func ModelTTS(
defer loader.Close()
if ttsModel == nil {
return "", nil, fmt.Errorf("could not load tts model %q", backendConfig.Model)
return "", nil, fmt.Errorf("could not load tts model %q", modelConfig.Model)
}
audioDir := filepath.Join(appConfig.GeneratedContentDir, "audio")
@@ -47,14 +47,14 @@ func ModelTTS(
// Checking first that it exists and is not outside ModelPath
// TODO: we should actually first check if the modelFile is looking like
// a FS path
mp := filepath.Join(loader.ModelPath, backendConfig.Model)
mp := filepath.Join(loader.ModelPath, modelConfig.Model)
if _, err := os.Stat(mp); err == nil {
if err := utils.VerifyPath(mp, appConfig.ModelPath); err != nil {
if err := utils.VerifyPath(mp, appConfig.SystemState.Model.ModelsPath); err != nil {
return "", nil, err
}
modelPath = mp
} else {
modelPath = backendConfig.Model // skip this step if it fails?????
modelPath = modelConfig.Model // skip this step if it fails?????
}
res, err := ttsModel.TTS(context.Background(), &proto.TTSRequest{
core/backend/vad.go
+2 -2
View File
@@ -13,8 +13,8 @@ func VAD(request *schema.VADRequest,
ctx context.Context,
ml *model.ModelLoader,
appConfig *config.ApplicationConfig,
backendConfig config.BackendConfig) (*schema.VADResponse, error) {
opts := ModelOptions(backendConfig, appConfig)
modelConfig config.ModelConfig) (*schema.VADResponse, error) {
opts := ModelOptions(modelConfig, appConfig)
vadModel, err := ml.Load(opts...)
if err != nil {
return nil, err
core/backend/video.go
+2 -2
View File
@@ -7,9 +7,9 @@ import (
model "github.com/mudler/LocalAI/pkg/model"
)
func VideoGeneration(height, width int32, prompt, startImage, endImage, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) {
func VideoGeneration(height, width int32, prompt, startImage, endImage, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
opts := ModelOptions(backendConfig, appConfig)
opts := ModelOptions(modelConfig, appConfig)
inferenceModel, err := loader.Load(
opts...,
)