mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-26 16:19:04 -04:00
feat: refactor build process, drop embedded backends (#5875)
* feat: split remaining backends and drop embedded backends - Drop silero-vad, huggingface, and stores backend from embedded binaries - Refactor Makefile and Dockerfile to avoid building grpc backends - Drop golang code that was used to embed backends - Simplify building by using goreleaser Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(gallery): be specific with llama-cpp backend templates Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(docs): update Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(ci): minor fixes Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore: drop all ffmpeg references Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix: run protogen-go Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Always enable p2p mode Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Update gorelease file Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(stores): do not always load Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix linting issues Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Simplify Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Mac OS fixup Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto authored and GitHub committed
1 parent
e29b2c3aff
commit
98e5291afc
118 files changed
+631
-1339
No files matched your search
+7
-121
@@ -5,18 +5,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||
"github.com/mudler/LocalAI/pkg/library"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
"github.com/phayes/freeport"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/elliotchance/orderedmap/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -51,79 +45,6 @@ const (
|
||||
LocalStoreBackend = "local-store"
|
||||
)
|
||||
|
||||
func backendPath(assetDir, backend string) string {
|
||||
return filepath.Join(assetDir, "backend-assets", "grpc", backend)
|
||||
}
|
||||
|
||||
// backendsInAssetDir returns the list of backends in the asset directory
|
||||
// that should be loaded
|
||||
func backendsInAssetDir(assetDir string) (map[string][]string, error) {
|
||||
// Exclude backends from automatic loading
|
||||
excludeBackends := []string{LocalStoreBackend}
|
||||
entry, err := os.ReadDir(backendPath(assetDir, ""))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backends := make(map[string][]string)
|
||||
ENTRY:
|
||||
for _, e := range entry {
|
||||
for _, exclude := range excludeBackends {
|
||||
if e.Name() == exclude {
|
||||
continue ENTRY
|
||||
}
|
||||
}
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(e.Name(), ".log") {
|
||||
continue
|
||||
}
|
||||
|
||||
backends[e.Name()] = []string{}
|
||||
}
|
||||
|
||||
return backends, nil
|
||||
}
|
||||
|
||||
func orderBackends(backends map[string][]string) ([]string, error) {
|
||||
// order backends from the asset directory.
|
||||
// as we scan for backends, we want to keep some order which backends are tried of.
|
||||
// for example, llama.cpp should be tried first, and we want to keep the huggingface backend at the last.
|
||||
|
||||
// sets a priority list - first has more priority
|
||||
priorityList := []string{}
|
||||
|
||||
toTheEnd := []string{
|
||||
// last has to be huggingface
|
||||
LCHuggingFaceBackend,
|
||||
}
|
||||
|
||||
// create an ordered map
|
||||
orderedBackends := orderedmap.NewOrderedMap[string, any]()
|
||||
// add priorityList first
|
||||
for _, p := range priorityList {
|
||||
if _, ok := backends[p]; ok {
|
||||
orderedBackends.Set(p, backends[p])
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range backends {
|
||||
if !slices.Contains(toTheEnd, k) {
|
||||
if _, ok := orderedBackends.Get(k); !ok {
|
||||
orderedBackends.Set(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, t := range toTheEnd {
|
||||
if _, ok := backends[t]; ok {
|
||||
orderedBackends.Set(t, backends[t])
|
||||
}
|
||||
}
|
||||
|
||||
return orderedBackends.Keys(), nil
|
||||
}
|
||||
|
||||
// starts the grpcModelProcess for the backend, and returns a grpc client
|
||||
// It also loads the model
|
||||
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
|
||||
@@ -177,35 +98,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
||||
client = NewModel(modelID, uri, nil)
|
||||
}
|
||||
} else {
|
||||
grpcProcess := backendPath(o.assetDir, backend)
|
||||
if err := utils.VerifyPath(grpcProcess, o.assetDir); err != nil {
|
||||
return nil, fmt.Errorf("referring to a backend not in asset dir: %s", err.Error())
|
||||
}
|
||||
|
||||
// Check if the file exists
|
||||
if _, err := os.Stat(grpcProcess); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("backend not found: %s", grpcProcess)
|
||||
}
|
||||
|
||||
serverAddress, err := getFreeAddress()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
|
||||
}
|
||||
|
||||
args := []string{}
|
||||
|
||||
// Load the ld.so if it exists
|
||||
args, grpcProcess = library.LoadLDSO(o.assetDir, args, grpcProcess)
|
||||
|
||||
// Make sure the process is executable in any circumstance
|
||||
process, err := ml.startProcess(grpcProcess, modelID, serverAddress, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug().Msgf("GRPC Service Started")
|
||||
|
||||
client = NewModel(modelID, serverAddress, process)
|
||||
return nil, fmt.Errorf("backend not found: %s", backend)
|
||||
}
|
||||
|
||||
log.Debug().Msgf("Wait for the service to start up")
|
||||
@@ -259,14 +152,6 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
|
||||
}
|
||||
}
|
||||
|
||||
func (ml *ModelLoader) ListAvailableBackends(assetdir string) ([]string, error) {
|
||||
backends, err := backendsInAssetDir(assetdir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orderBackends(backends)
|
||||
}
|
||||
|
||||
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
|
||||
o := NewOptions(opts...)
|
||||
|
||||
@@ -346,17 +231,18 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
|
||||
var err error
|
||||
|
||||
// get backends embedded in the binary
|
||||
autoLoadBackends, err := ml.ListAvailableBackends(o.assetDir)
|
||||
if err != nil {
|
||||
ml.Close() // we failed, release the lock
|
||||
return nil, err
|
||||
}
|
||||
autoLoadBackends := []string{}
|
||||
|
||||
// append externalBackends supplied by the user via the CLI
|
||||
for b := range ml.GetAllExternalBackends(o) {
|
||||
autoLoadBackends = append(autoLoadBackends, b)
|
||||
}
|
||||
|
||||
if len(autoLoadBackends) == 0 {
|
||||
log.Error().Msg("No backends found")
|
||||
return nil, fmt.Errorf("no backends found")
|
||||
}
|
||||
|
||||
log.Debug().Msgf("Loading from the following backends (in order): %+v", autoLoadBackends)
|
||||
|
||||
log.Info().Msgf("Trying to load the model '%s' with the backend '%s'", o.modelID, autoLoadBackends)
|
||||
|
||||
@@ -10,7 +10,6 @@ type Options struct {
|
||||
backendString string
|
||||
model string
|
||||
modelID string
|
||||
assetDir string
|
||||
context context.Context
|
||||
|
||||
gRPCOptions *pb.ModelOptions
|
||||
@@ -75,12 +74,6 @@ func WithLoadGRPCLoadModelOpts(opts *pb.ModelOptions) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithAssetDir(assetDir string) Option {
|
||||
return func(o *Options) {
|
||||
o.assetDir = assetDir
|
||||
}
|
||||
}
|
||||
|
||||
func WithContext(ctx context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.context = ctx
|
||||
|
||||
Reference in new issue
Block a user