feat(backends): add system backend, refactor (#6059)

- Add a system backend path
- Refactor and consolidate system information in system state
- Use system state in all the components to figure out the system paths
  to used whenever needed
- Refactor BackendConfig -> ModelConfig. This was otherway misleading as
  now we do have a backend configuration which is not the model config.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/gallery/backends.go

@@ -59,10 +59,10 @@ func writeBackendMetadata(backendPath string, metadata *BackendMetadata) error {
 }
 
 // Installs a model from the gallery
-func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.SystemState, name string, basePath string, downloadStatus func(string, string, string, float64), force bool) error {
+func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.SystemState, name string, downloadStatus func(string, string, string, float64), force bool) error {
 	if !force {
 		// check if we already have the backend installed
-		backends, err := ListSystemBackends(basePath)
+		backends, err := ListSystemBackends(systemState)
 		if err != nil {
 			return err
 		}
@@ -77,12 +77,12 @@ func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.S
 
 	log.Debug().Interface("galleries", galleries).Str("name", name).Msg("Installing backend from gallery")
 
-	backends, err := AvailableBackends(galleries, basePath)
+	backends, err := AvailableBackends(galleries, systemState)
 	if err != nil {
 		return err
 	}
 
-	backend := FindGalleryElement(backends, name, basePath)
+	backend := FindGalleryElement(backends, name)
 	if backend == nil {
 		return fmt.Errorf("no backend found with name %q", name)
 	}
@@ -99,12 +99,12 @@ func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.S
 		log.Debug().Str("name", name).Str("bestBackend", bestBackend.Name).Msg("Installing backend from meta backend")
 
 		// Then, let's install the best backend
-		if err := InstallBackend(basePath, bestBackend, downloadStatus); err != nil {
+		if err := InstallBackend(systemState, bestBackend, downloadStatus); err != nil {
 			return err
 		}
 
 		// we need now to create a path for the meta backend, with the alias to the installed ones so it can be used to remove it
-		metaBackendPath := filepath.Join(basePath, name)
+		metaBackendPath := filepath.Join(systemState.Backend.BackendsPath, name)
 		if err := os.MkdirAll(metaBackendPath, 0750); err != nil {
 			return fmt.Errorf("failed to create meta backend path %q: %v", metaBackendPath, err)
 		}
@@ -124,12 +124,12 @@ func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.S
 		return nil
 	}
 
-	return InstallBackend(basePath, backend, downloadStatus)
+	return InstallBackend(systemState, backend, downloadStatus)
 }
 
-func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error {
+func InstallBackend(systemState *system.SystemState, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error {
 	// Create base path if it doesn't exist
-	err := os.MkdirAll(basePath, 0750)
+	err := os.MkdirAll(systemState.Backend.BackendsPath, 0750)
 	if err != nil {
 		return fmt.Errorf("failed to create base path: %v", err)
 	}
@@ -139,7 +139,7 @@ func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func
 	}
 
 	name := config.Name
-	backendPath := filepath.Join(basePath, name)
+	backendPath := filepath.Join(systemState.Backend.BackendsPath, name)
 	err = os.MkdirAll(backendPath, 0750)
 	if err != nil {
 		return fmt.Errorf("failed to create base path: %v", err)
@@ -188,14 +188,28 @@ func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func
 	return nil
 }
 
-func DeleteBackendFromSystem(basePath string, name string) error {
-	backendDirectory := filepath.Join(basePath, name)
+func DeleteBackendFromSystem(systemState *system.SystemState, name string) error {
+	backends, err := ListSystemBackends(systemState)
+	if err != nil {
+		return err
+	}
+
+	backend, ok := backends.Get(name)
+	if !ok {
+		return fmt.Errorf("backend %q not found", name)
+	}
+
+	if backend.IsSystem {
+		return fmt.Errorf("system backend %q cannot be deleted", name)
+	}
+
+	backendDirectory := filepath.Join(systemState.Backend.BackendsPath, name)
 
 	// check if the backend dir exists
 	if _, err := os.Stat(backendDirectory); os.IsNotExist(err) {
 		// if doesn't exist, it might be an alias, so we need to check if we have a matching alias in
 		// all the backends in the basePath
-		backends, err := os.ReadDir(basePath)
+		backends, err := os.ReadDir(systemState.Backend.BackendsPath)
 		if err != nil {
 			return err
 		}
@@ -203,12 +217,12 @@ func DeleteBackendFromSystem(basePath string, name string) error {
 
 		for _, backend := range backends {
 			if backend.IsDir() {
-				metadata, err := readBackendMetadata(filepath.Join(basePath, backend.Name()))
+				metadata, err := readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, backend.Name()))
 				if err != nil {
 					return err
 				}
 				if metadata != nil && metadata.Alias == name {
-					backendDirectory = filepath.Join(basePath, backend.Name())
+					backendDirectory = filepath.Join(systemState.Backend.BackendsPath, backend.Name())
 					foundBackend = true
 					break
 				}
@@ -228,7 +242,7 @@ func DeleteBackendFromSystem(basePath string, name string) error {
 	}
 
 	if metadata != nil && metadata.MetaBackendFor != "" {
-		metaBackendDirectory := filepath.Join(basePath, metadata.MetaBackendFor)
+		metaBackendDirectory := filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor)
 		log.Debug().Str("backendDirectory", metaBackendDirectory).Msg("Deleting meta backend")
 		if _, err := os.Stat(metaBackendDirectory); os.IsNotExist(err) {
 			return fmt.Errorf("meta backend %q not found", metadata.MetaBackendFor)
@@ -243,6 +257,7 @@ type SystemBackend struct {
 	Name     string
 	RunFile  string
 	IsMeta   bool
+	IsSystem bool
 	Metadata *BackendMetadata
 }
 
@@ -266,30 +281,51 @@ func (b SystemBackends) GetAll() []SystemBackend {
 	return backends
 }
 
-func ListSystemBackends(basePath string) (SystemBackends, error) {
-	potentialBackends, err := os.ReadDir(basePath)
+func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) {
+	potentialBackends, err := os.ReadDir(systemState.Backend.BackendsPath)
 	if err != nil {
 		return nil, err
 	}
 
 	backends := make(SystemBackends)
 
+	systemBackends, err := os.ReadDir(systemState.Backend.BackendsSystemPath)
+	if err == nil {
+		// system backends are special, they are provided by the system and not managed by LocalAI
+		for _, systemBackend := range systemBackends {
+			if systemBackend.IsDir() {
+				systemBackendRunFile := filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile)
+				if _, err := os.Stat(systemBackendRunFile); err == nil {
+					backends[systemBackend.Name()] = SystemBackend{
+						Name:     systemBackend.Name(),
+						RunFile:  filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile),
+						IsMeta:   false,
+						IsSystem: true,
+						Metadata: nil,
+					}
+				}
+			}
+		}
+	} else {
+		log.Warn().Err(err).Msg("Failed to read system backends, but that's ok, we will just use the backends managed by LocalAI")
+	}
+
 	for _, potentialBackend := range potentialBackends {
 		if potentialBackend.IsDir() {
-			potentialBackendRunFile := filepath.Join(basePath, potentialBackend.Name(), runFile)
+			potentialBackendRunFile := filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name(), runFile)
 
 			var metadata *BackendMetadata
 
 			// If metadata file does not exist, we just use the directory name
 			// and we do not fill the other metadata (such as potential backend Aliases)
-			metadataFilePath := filepath.Join(basePath, potentialBackend.Name(), metadataFile)
+			metadataFilePath := filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name(), metadataFile)
 			if _, err := os.Stat(metadataFilePath); os.IsNotExist(err) {
 				metadata = &BackendMetadata{
 					Name: potentialBackend.Name(),
 				}
 			} else {
 				// Check for alias in metadata
-				metadata, err = readBackendMetadata(filepath.Join(basePath, potentialBackend.Name()))
+				metadata, err = readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name()))
 				if err != nil {
 					return nil, err
 				}
@@ -323,7 +359,7 @@ func ListSystemBackends(basePath string) (SystemBackends, error) {
 			if metadata.MetaBackendFor != "" {
 				backends[metadata.Name] = SystemBackend{
 					Name:     metadata.Name,
-					RunFile:  filepath.Join(basePath, metadata.MetaBackendFor, runFile),
+					RunFile:  filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor, runFile),
 					IsMeta:   true,
 					Metadata: metadata,
 				}
@@ -334,8 +370,8 @@ func ListSystemBackends(basePath string) (SystemBackends, error) {
 	return backends, nil
 }
 
-func RegisterBackends(basePath string, modelLoader *model.ModelLoader) error {
-	backends, err := ListSystemBackends(basePath)
+func RegisterBackends(systemState *system.SystemState, modelLoader *model.ModelLoader) error {
+	backends, err := ListSystemBackends(systemState)
 	if err != nil {
 		return err
 	}

core/gallery/backends_test.go

@@ -43,13 +43,21 @@ var _ = Describe("Gallery Backends", func() {
 
 	Describe("InstallBackendFromGallery", func() {
 		It("should return error when backend is not found", func() {
-			err := InstallBackendFromGallery(galleries, nil, "non-existent", tempDir, nil, true)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = InstallBackendFromGallery(galleries, systemState, "non-existent", nil, true)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no backend found with name \"non-existent\""))
 		})
 
 		It("should install backend from gallery", func() {
-			err := InstallBackendFromGallery(galleries, nil, "test-backend", tempDir, nil, true)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = InstallBackendFromGallery(galleries, systemState, "test-backend", nil, true)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
 		})
@@ -220,26 +228,32 @@ var _ = Describe("Gallery Backends", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// Test with NVIDIA system state
-			nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000}
-			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", tempDir, nil, true)
+			nvidiaSystemState := &system.SystemState{
+				GPUVendor: "nvidia",
+				VRAM:      1000000000000,
+				Backend:   system.Backend{BackendsPath: tempDir},
+			}
+			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true)
 			Expect(err).NotTo(HaveOccurred())
 
 			metaBackendPath := filepath.Join(tempDir, "meta-backend")
 			Expect(metaBackendPath).To(BeADirectory())
 
-			metaBackendPath = filepath.Join(tempDir, "meta-backend", "metadata.json")
-			Expect(metaBackendPath).To(BeARegularFile())
-
 			concreteBackendPath := filepath.Join(tempDir, "nvidia-backend")
 			Expect(concreteBackendPath).To(BeADirectory())
 
-			allBackends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(allBackends.Exists("meta-backend")).To(BeTrue())
-			Expect(allBackends.Exists("nvidia-backend")).To(BeTrue())
+
+			allBackends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(allBackends).To(HaveKey("meta-backend"))
+			Expect(allBackends).To(HaveKey("nvidia-backend"))
 
 			// Delete meta backend by name
-			err = DeleteBackendFromSystem(tempDir, "meta-backend")
+			err = DeleteBackendFromSystem(systemState, "meta-backend")
 			Expect(err).NotTo(HaveOccurred())
 
 			// Verify meta backend directory is deleted
@@ -294,8 +308,12 @@ var _ = Describe("Gallery Backends", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// Test with NVIDIA system state
-			nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000}
-			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", tempDir, nil, true)
+			nvidiaSystemState := &system.SystemState{
+				GPUVendor: "nvidia",
+				VRAM:      1000000000000,
+				Backend:   system.Backend{BackendsPath: tempDir},
+			}
+			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true)
 			Expect(err).NotTo(HaveOccurred())
 
 			metaBackendPath := filepath.Join(tempDir, "meta-backend")
@@ -304,19 +322,22 @@ var _ = Describe("Gallery Backends", func() {
 			concreteBackendPath := filepath.Join(tempDir, "nvidia-backend")
 			Expect(concreteBackendPath).To(BeADirectory())
 
-			allBackends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(allBackends.Exists("meta-backend")).To(BeTrue())
-			Expect(allBackends.Exists("nvidia-backend")).To(BeTrue())
 
-			backend, ok := allBackends.Get("meta-backend")
-			Expect(ok).To(BeTrue())
-			Expect(backend.Metadata.MetaBackendFor).To(Equal("nvidia-backend"))
-			Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, "nvidia-backend", "run.sh")))
-			Expect(backend.IsMeta).To(BeTrue())
+			allBackends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(allBackends).To(HaveKey("meta-backend"))
+			Expect(allBackends).To(HaveKey("nvidia-backend"))
+			mback, exists := allBackends.Get("meta-backend")
+			Expect(exists).To(BeTrue())
+			Expect(mback.IsMeta).To(BeTrue())
+			Expect(mback.Metadata.MetaBackendFor).To(Equal("nvidia-backend"))
 
 			// Delete meta backend by name
-			err = DeleteBackendFromSystem(tempDir, "meta-backend")
+			err = DeleteBackendFromSystem(systemState, "meta-backend")
 			Expect(err).NotTo(HaveOccurred())
 
 			// Verify meta backend directory is deleted
@@ -371,8 +392,12 @@ var _ = Describe("Gallery Backends", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// Test with NVIDIA system state
-			nvidiaSystemState := &system.SystemState{GPUVendor: "nvidia", VRAM: 1000000000000}
-			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", tempDir, nil, true)
+			nvidiaSystemState := &system.SystemState{
+				GPUVendor: "nvidia",
+				VRAM:      1000000000000,
+				Backend:   system.Backend{BackendsPath: tempDir},
+			}
+			err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true)
 			Expect(err).NotTo(HaveOccurred())
 
 			metaBackendPath := filepath.Join(tempDir, "meta-backend")
@@ -381,16 +406,21 @@ var _ = Describe("Gallery Backends", func() {
 			concreteBackendPath := filepath.Join(tempDir, "nvidia-backend")
 			Expect(concreteBackendPath).To(BeADirectory())
 
-			allBackends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(allBackends.Exists("meta-backend")).To(BeTrue())
-			Expect(allBackends.Exists("nvidia-backend")).To(BeTrue())
-			backend, ok := allBackends.Get("meta-backend")
-			Expect(ok).To(BeTrue())
-			Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, "nvidia-backend", "run.sh")))
+
+			allBackends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(allBackends).To(HaveKey("meta-backend"))
+			Expect(allBackends).To(HaveKey("nvidia-backend"))
+			mback, exists := allBackends.Get("meta-backend")
+			Expect(exists).To(BeTrue())
+			Expect(mback.RunFile).To(Equal(filepath.Join(tempDir, "nvidia-backend", "run.sh")))
 
 			// Delete meta backend by name
-			err = DeleteBackendFromSystem(tempDir, "meta-backend")
+			err = DeleteBackendFromSystem(systemState, "meta-backend")
 			Expect(err).NotTo(HaveOccurred())
 
 			// Verify meta backend directory is deleted
@@ -427,25 +457,28 @@ var _ = Describe("Gallery Backends", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			// List system backends
-			backends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
 
+			backends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+
+			metaBackend, exists := backends.Get("meta-backend")
+			concreteBackendRunFile := filepath.Join(tempDir, "concrete-backend", "run.sh")
+
 			// Should include both the meta backend name and concrete backend name
-			Expect(backends.Exists("meta-backend")).To(BeTrue())
+			Expect(exists).To(BeTrue())
 			Expect(backends.Exists("concrete-backend")).To(BeTrue())
 
-			// meta-backend should point to concrete-backend
-			Expect(backends.Exists("meta-backend")).To(BeTrue())
-			backend, ok := backends.Get("meta-backend")
-			Expect(ok).To(BeTrue())
-			Expect(backend.Metadata.MetaBackendFor).To(Equal("concrete-backend"))
-			Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, "concrete-backend", "run.sh")))
-			Expect(backend.IsMeta).To(BeTrue())
-
+			// meta-backend should be empty
+			Expect(metaBackend.IsMeta).To(BeTrue())
+			Expect(metaBackend.RunFile).To(Equal(concreteBackendRunFile))
 			// concrete-backend should point to its own run.sh
-			backend, ok = backends.Get("concrete-backend")
-			Expect(ok).To(BeTrue())
-			Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, "concrete-backend", "run.sh")))
+			concreteBackend, exists := backends.Get("concrete-backend")
+			Expect(exists).To(BeTrue())
+			Expect(concreteBackend.RunFile).To(Equal(concreteBackendRunFile))
 		})
 	})
 
@@ -459,11 +492,80 @@ var _ = Describe("Gallery Backends", func() {
 				URI: "test-uri",
 			}
 
-			err := InstallBackend(newPath, &backend, nil)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(newPath),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = InstallBackend(systemState, &backend, nil)
 			Expect(err).To(HaveOccurred()) // Will fail due to invalid URI, but path should be created
 			Expect(newPath).To(BeADirectory())
 		})
 
+		It("should overwrite existing backend", func() {
+			if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+				Skip("Skipping test on darwin/arm64")
+			}
+			newPath := filepath.Join(tempDir, "test-backend")
+
+			// Create a dummy backend directory
+			err := os.MkdirAll(newPath, 0750)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = os.WriteFile(filepath.Join(newPath, "metadata.json"), []byte("foo"), 0644)
+			Expect(err).NotTo(HaveOccurred())
+			err = os.WriteFile(filepath.Join(newPath, "run.sh"), []byte(""), 0644)
+			Expect(err).NotTo(HaveOccurred())
+
+			backend := GalleryBackend{
+				Metadata: Metadata{
+					Name: "test-backend",
+				},
+				URI:   "quay.io/mudler/tests:localai-backend-test",
+				Alias: "test-alias",
+			}
+
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = InstallBackend(systemState, &backend, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile())
+			dat, err := os.ReadFile(filepath.Join(tempDir, "test-backend", "metadata.json"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(dat)).ToNot(Equal("foo"))
+		})
+
+		It("should overwrite existing backend", func() {
+			if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+				Skip("Skipping test on darwin/arm64")
+			}
+			newPath := filepath.Join(tempDir, "test-backend")
+
+			// Create a dummy backend directory
+			err := os.MkdirAll(newPath, 0750)
+			Expect(err).NotTo(HaveOccurred())
+
+			backend := GalleryBackend{
+				Metadata: Metadata{
+					Name: "test-backend",
+				},
+				URI:   "quay.io/mudler/tests:localai-backend-test",
+				Alias: "test-alias",
+			}
+
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).ToNot(BeARegularFile())
+
+			err = InstallBackend(systemState, &backend, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile())
+		})
+
 		It("should create alias file when specified", func() {
 			if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
 				Skip("Skipping test on darwin/arm64")
@@ -476,7 +578,11 @@ var _ = Describe("Gallery Backends", func() {
 				Alias: "test-alias",
 			}
 
-			err := InstallBackend(tempDir, &backend, nil)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = InstallBackend(systemState, &backend, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile())
 
@@ -492,16 +598,14 @@ var _ = Describe("Gallery Backends", func() {
 			Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
 
 			// Check that the alias was recognized
-			backends, err := ListSystemBackends(tempDir)
+			backends, err := ListSystemBackends(systemState)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(backends.Exists("test-alias")).To(BeTrue())
-			Expect(backends.Exists("test-backend")).To(BeTrue())
-			b, ok := backends.Get("test-alias")
-			Expect(ok).To(BeTrue())
-			Expect(b.RunFile).To(Equal(filepath.Join(tempDir, "test-backend", "run.sh")))
-			b, ok = backends.Get("test-backend")
-			Expect(ok).To(BeTrue())
-			Expect(b.RunFile).To(Equal(filepath.Join(tempDir, "test-backend", "run.sh")))
+			aliasBackend, exists := backends.Get("test-alias")
+			Expect(exists).To(BeTrue())
+			Expect(aliasBackend.RunFile).To(Equal(filepath.Join(tempDir, "test-backend", "run.sh")))
+			testB, exists := backends.Get("test-backend")
+			Expect(exists).To(BeTrue())
+			Expect(testB.RunFile).To(Equal(filepath.Join(tempDir, "test-backend", "run.sh")))
 		})
 	})
 
@@ -514,13 +618,26 @@ var _ = Describe("Gallery Backends", func() {
 			err := os.MkdirAll(backendPath, 0750)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = DeleteBackendFromSystem(tempDir, backendName)
+			err = os.WriteFile(filepath.Join(backendPath, "metadata.json"), []byte("{}"), 0644)
+			Expect(err).NotTo(HaveOccurred())
+			err = os.WriteFile(filepath.Join(backendPath, "run.sh"), []byte(""), 0644)
+			Expect(err).NotTo(HaveOccurred())
+
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = DeleteBackendFromSystem(systemState, backendName)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(backendPath).NotTo(BeADirectory())
 		})
 
 		It("should not error when backend doesn't exist", func() {
-			err := DeleteBackendFromSystem(tempDir, "non-existent")
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			err = DeleteBackendFromSystem(systemState, "non-existent")
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -538,14 +655,17 @@ var _ = Describe("Gallery Backends", func() {
 				Expect(err).NotTo(HaveOccurred())
 			}
 
-			backends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(backends.GetAll()).To(HaveLen(len(backendNames)))
+			backends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(backends).To(HaveLen(len(backendNames)))
 
 			for _, name := range backendNames {
-				Expect(backends.Exists(name)).To(BeTrue())
-				backend, ok := backends.Get(name)
-				Expect(ok).To(BeTrue())
+				backend, exists := backends.Get(name)
+				Expect(exists).To(BeTrue())
 				Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, name, "run.sh")))
 			}
 		})
@@ -572,16 +692,23 @@ var _ = Describe("Gallery Backends", func() {
 			err = os.WriteFile(filepath.Join(backendPath, "run.sh"), []byte(""), 0755)
 			Expect(err).NotTo(HaveOccurred())
 
-			backends, err := ListSystemBackends(tempDir)
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath(tempDir),
+			)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(backends.Exists(alias)).To(BeTrue())
-			backend, ok := backends.Get(alias)
-			Expect(ok).To(BeTrue())
+			backends, err := ListSystemBackends(systemState)
+			Expect(err).NotTo(HaveOccurred())
+			backend, exists := backends.Get(alias)
+			Expect(exists).To(BeTrue())
 			Expect(backend.RunFile).To(Equal(filepath.Join(tempDir, backendName, "run.sh")))
 		})
 
 		It("should return error when base path doesn't exist", func() {
-			_, err := ListSystemBackends(filepath.Join(tempDir, "non-existent"))
+			systemState, err := system.GetSystemState(
+				system.WithBackendPath("foobardir"),
+			)
+			Expect(err).NotTo(HaveOccurred())
+			_, err = ListSystemBackends(systemState)
 			Expect(err).To(HaveOccurred())
 		})
 	})

core/gallery/gallery.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/downloader"
+	"github.com/mudler/LocalAI/pkg/system"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v2"
 )
@@ -89,7 +90,7 @@ func (gm GalleryElements[T]) Paginate(pageNum int, itemsNum int) GalleryElements
 	return gm[start:end]
 }
 
-func FindGalleryElement[T GalleryElement](models []T, name string, basePath string) T {
+func FindGalleryElement[T GalleryElement](models []T, name string) T {
 	var model T
 	name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
 
@@ -116,13 +117,13 @@ func FindGalleryElement[T GalleryElement](models []T, name string, basePath stri
 // List available models
 // Models galleries are a list of yaml files that are hosted on a remote server (for example github).
 // Each yaml file contains a list of models that can be downloaded and optionally overrides to define a new model setting.
-func AvailableGalleryModels(galleries []config.Gallery, basePath string) (GalleryElements[*GalleryModel], error) {
+func AvailableGalleryModels(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryModel], error) {
 	var models []*GalleryModel
 
 	// Get models from galleries
 	for _, gallery := range galleries {
-		galleryModels, err := getGalleryElements[*GalleryModel](gallery, basePath, func(model *GalleryModel) bool {
-			if _, err := os.Stat(filepath.Join(basePath, fmt.Sprintf("%s.yaml", model.GetName()))); err == nil {
+		galleryModels, err := getGalleryElements[*GalleryModel](gallery, systemState.Model.ModelsPath, func(model *GalleryModel) bool {
+			if _, err := os.Stat(filepath.Join(systemState.Model.ModelsPath, fmt.Sprintf("%s.yaml", model.GetName()))); err == nil {
 				return true
 			}
 			return false
@@ -137,13 +138,13 @@ func AvailableGalleryModels(galleries []config.Gallery, basePath string) (Galler
 }
 
 // List available backends
-func AvailableBackends(galleries []config.Gallery, basePath string) (GalleryElements[*GalleryBackend], error) {
-	var models []*GalleryBackend
+func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
+	var backends []*GalleryBackend
 
-	// Get models from galleries
+	// Get backends from galleries
 	for _, gallery := range galleries {
-		galleryModels, err := getGalleryElements[*GalleryBackend](gallery, basePath, func(backend *GalleryBackend) bool {
-			backends, err := ListSystemBackends(basePath)
+		galleryBackends, err := getGalleryElements[*GalleryBackend](gallery, systemState.Backend.BackendsPath, func(backend *GalleryBackend) bool {
+			backends, err := ListSystemBackends(systemState)
 			if err != nil {
 				return false
 			}
@@ -152,10 +153,10 @@ func AvailableBackends(galleries []config.Gallery, basePath string) (GalleryElem
 		if err != nil {
 			return nil, err
 		}
-		models = append(models, galleryModels...)
+		backends = append(backends, galleryBackends...)
 	}
 
-	return models, nil
+	return backends, nil
 }
 
 func findGalleryURLFromReferenceURL(url string, basePath string) (string, error) {

core/gallery/models.go

@@ -72,7 +72,8 @@ type PromptTemplate struct {
 // Installs a model from the gallery
 func InstallModelFromGallery(
 	modelGalleries, backendGalleries []config.Gallery,
-	name string, basePath, backendBasePath string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool) error {
+	systemState *system.SystemState,
+	name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool) error {
 
 	applyModel := func(model *GalleryModel) error {
 		name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
@@ -81,7 +82,7 @@ func InstallModelFromGallery(
 
 		if len(model.URL) > 0 {
 			var err error
-			config, err = GetGalleryConfigFromURL[ModelConfig](model.URL, basePath)
+			config, err = GetGalleryConfigFromURL[ModelConfig](model.URL, systemState.Model.ModelsPath)
 			if err != nil {
 				return err
 			}
@@ -122,19 +123,15 @@ func InstallModelFromGallery(
 			return err
 		}
 
-		installedModel, err := InstallModel(basePath, installName, &config, model.Overrides, downloadStatus, enforceScan)
+		installedModel, err := InstallModel(systemState, installName, &config, model.Overrides, downloadStatus, enforceScan)
 		if err != nil {
 			return err
 		}
 		log.Debug().Msgf("Installed model %q", installedModel.Name)
 		if automaticallyInstallBackend && installedModel.Backend != "" {
 			log.Debug().Msgf("Installing backend %q", installedModel.Backend)
-			systemState, err := system.GetSystemState()
-			if err != nil {
-				return err
-			}
 
-			if err := InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, backendBasePath, downloadStatus, false); err != nil {
+			if err := InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, downloadStatus, false); err != nil {
 				return err
 			}
 		}
@@ -142,12 +139,12 @@ func InstallModelFromGallery(
 		return nil
 	}
 
-	models, err := AvailableGalleryModels(modelGalleries, basePath)
+	models, err := AvailableGalleryModels(modelGalleries, systemState)
 	if err != nil {
 		return err
 	}
 
-	model := FindGalleryElement(models, name, basePath)
+	model := FindGalleryElement(models, name)
 	if model == nil {
 		return fmt.Errorf("no model found with name %q", name)
 	}
@@ -155,7 +152,8 @@ func InstallModelFromGallery(
 	return applyModel(model)
 }
 
-func InstallModel(basePath, nameOverride string, config *ModelConfig, configOverrides map[string]interface{}, downloadStatus func(string, string, string, float64), enforceScan bool) (*lconfig.BackendConfig, error) {
+func InstallModel(systemState *system.SystemState, nameOverride string, config *ModelConfig, configOverrides map[string]interface{}, downloadStatus func(string, string, string, float64), enforceScan bool) (*lconfig.ModelConfig, error) {
+	basePath := systemState.Model.ModelsPath
 	// Create base path if it doesn't exist
 	err := os.MkdirAll(basePath, 0750)
 	if err != nil {
@@ -221,7 +219,7 @@ func InstallModel(basePath, nameOverride string, config *ModelConfig, configOver
 		return nil, err
 	}
 
-	backendConfig := lconfig.BackendConfig{}
+	modelConfig := lconfig.ModelConfig{}
 
 	// write config file
 	if len(configOverrides) != 0 || len(config.ConfigFile) != 0 {
@@ -246,12 +244,12 @@ func InstallModel(basePath, nameOverride string, config *ModelConfig, configOver
 			return nil, fmt.Errorf("failed to marshal updated config YAML: %v", err)
 		}
 
-		err = yaml.Unmarshal(updatedConfigYAML, &backendConfig)
+		err = yaml.Unmarshal(updatedConfigYAML, &modelConfig)
 		if err != nil {
 			return nil, fmt.Errorf("failed to unmarshal updated config YAML: %v", err)
 		}
 
-		if !backendConfig.Validate() {
+		if !modelConfig.Validate() {
 			return nil, fmt.Errorf("failed to validate updated config YAML")
 		}
 
@@ -272,7 +270,7 @@ func InstallModel(basePath, nameOverride string, config *ModelConfig, configOver
 
 	log.Debug().Msgf("Written gallery file %s", modelFile)
 
-	return &backendConfig, os.WriteFile(modelFile, data, 0600)
+	return &modelConfig, os.WriteFile(modelFile, data, 0600)
 }
 
 func galleryFileName(name string) string {
@@ -285,21 +283,39 @@ func GetLocalModelConfiguration(basePath string, name string) (*ModelConfig, err
 	return ReadConfigFile[ModelConfig](galleryFile)
 }
 
-func DeleteModelFromSystem(basePath string, name string, additionalFiles []string) error {
-	// os.PathSeparator is not allowed in model names. Replace them with "__" to avoid conflicts with file paths.
-	name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
+func DeleteModelFromSystem(systemState *system.SystemState, name string) error {
+	additionalFiles := []string{}
 
-	configFile := filepath.Join(basePath, fmt.Sprintf("%s.yaml", name))
+	configFile := filepath.Join(systemState.Model.ModelsPath, fmt.Sprintf("%s.yaml", name))
+	if err := utils.VerifyPath(configFile, systemState.Model.ModelsPath); err != nil {
+		return fmt.Errorf("failed to verify path %s: %w", configFile, err)
+	}
+	// Galleryname is the name of the model in this case
+	dat, err := os.ReadFile(configFile)
+	if err == nil {
+		modelConfig := &config.ModelConfig{}
 
-	galleryFile := filepath.Join(basePath, galleryFileName(name))
+		err = yaml.Unmarshal(dat, &modelConfig)
+		if err != nil {
+			return err
+		}
+		if modelConfig.Model != "" {
+			additionalFiles = append(additionalFiles, modelConfig.ModelFileName())
+		}
 
-	for _, f := range []string{configFile, galleryFile} {
-		if err := utils.VerifyPath(f, basePath); err != nil {
-			return fmt.Errorf("failed to verify path %s: %w", f, err)
+		if modelConfig.MMProj != "" {
+			additionalFiles = append(additionalFiles, modelConfig.MMProjFileName())
 		}
 	}
 
-	var err error
+	// os.PathSeparator is not allowed in model names. Replace them with "__" to avoid conflicts with file paths.
+	name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
+
+	galleryFile := filepath.Join(systemState.Model.ModelsPath, galleryFileName(name))
+	if err := utils.VerifyPath(galleryFile, systemState.Model.ModelsPath); err != nil {
+		return fmt.Errorf("failed to verify path %s: %w", galleryFile, err)
+	}
+
 	// Delete all the files associated to the model
 	// read the model config
 	galleryconfig, err := ReadConfigFile[ModelConfig](galleryFile)
@@ -312,13 +328,19 @@ func DeleteModelFromSystem(basePath string, name string, additionalFiles []strin
 	// Remove additional files
 	if galleryconfig != nil {
 		for _, f := range galleryconfig.Files {
-			fullPath := filepath.Join(basePath, f.Filename)
+			fullPath := filepath.Join(systemState.Model.ModelsPath, f.Filename)
+			if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
+				return fmt.Errorf("failed to verify path %s: %w", fullPath, err)
+			}
 			filesToRemove = append(filesToRemove, fullPath)
 		}
 	}
 
 	for _, f := range additionalFiles {
-		fullPath := filepath.Join(filepath.Join(basePath, f))
+		fullPath := filepath.Join(filepath.Join(systemState.Model.ModelsPath, f))
+		if err := utils.VerifyPath(fullPath, systemState.Model.ModelsPath); err != nil {
+			return fmt.Errorf("failed to verify path %s: %w", fullPath, err)
+		}
 		filesToRemove = append(filesToRemove, fullPath)
 	}
 
@@ -340,8 +362,8 @@ func DeleteModelFromSystem(basePath string, name string, additionalFiles []strin
 
 // This is ***NEVER*** going to be perfect or finished.
 // This is a BEST EFFORT function to surface known-vulnerable models to users.
-func SafetyScanGalleryModels(galleries []config.Gallery, basePath string) error {
-	galleryModels, err := AvailableGalleryModels(galleries, basePath)
+func SafetyScanGalleryModels(galleries []config.Gallery, systemState *system.SystemState) error {
+	galleryModels, err := AvailableGalleryModels(galleries, systemState)
 	if err != nil {
 		return err
 	}

core/gallery/models_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/mudler/LocalAI/core/config"
 	. "github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"gopkg.in/yaml.v3"
@@ -29,7 +30,11 @@ var _ = Describe("Model test", func() {
 			defer os.RemoveAll(tempdir)
 			c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
 			Expect(err).ToNot(HaveOccurred())
-			_, err = InstallModel(tempdir, "", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
+			systemState, err := system.GetSystemState(
+				system.WithModelPath(tempdir),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = InstallModel(systemState, "", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			for _, f := range []string{"cerebras", "cerebras-completion.tmpl", "cerebras-chat.tmpl", "cerebras.yaml"} {
@@ -71,15 +76,19 @@ var _ = Describe("Model test", func() {
 					URL:  "file://" + galleryFilePath,
 				},
 			}
+			systemState, err := system.GetSystemState(
+				system.WithModelPath(tempdir),
+			)
+			Expect(err).ToNot(HaveOccurred())
 
-			models, err := AvailableGalleryModels(galleries, tempdir)
+			models, err := AvailableGalleryModels(galleries, systemState)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(models)).To(Equal(1))
 			Expect(models[0].Name).To(Equal("bert"))
 			Expect(models[0].URL).To(Equal(bertEmbeddingsURL))
 			Expect(models[0].Installed).To(BeFalse())
 
-			err = InstallModelFromGallery(galleries, []config.Gallery{}, "test@bert", tempdir, "", GalleryModel{}, func(s1, s2, s3 string, f float64) {}, true, true)
+			err = InstallModelFromGallery(galleries, []config.Gallery{}, systemState, "test@bert", GalleryModel{}, func(s1, s2, s3 string, f float64) {}, true, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			dat, err := os.ReadFile(filepath.Join(tempdir, "bert.yaml"))
@@ -90,16 +99,16 @@ var _ = Describe("Model test", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(content["usage"]).To(ContainSubstring("You can test this model with curl like this"))
 
-			models, err = AvailableGalleryModels(galleries, tempdir)
+			models, err = AvailableGalleryModels(galleries, systemState)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(models)).To(Equal(1))
 			Expect(models[0].Installed).To(BeTrue())
 
 			// delete
-			err = DeleteModelFromSystem(tempdir, "bert", []string{})
+			err = DeleteModelFromSystem(systemState, "bert")
 			Expect(err).ToNot(HaveOccurred())
 
-			models, err = AvailableGalleryModels(galleries, tempdir)
+			models, err = AvailableGalleryModels(galleries, systemState)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(models)).To(Equal(1))
 			Expect(models[0].Installed).To(BeFalse())
@@ -116,7 +125,11 @@ var _ = Describe("Model test", func() {
 			c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = InstallModel(tempdir, "foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
+			systemState, err := system.GetSystemState(
+				system.WithModelPath(tempdir),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = InstallModel(systemState, "foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			for _, f := range []string{"cerebras", "cerebras-completion.tmpl", "cerebras-chat.tmpl", "foo.yaml"} {
@@ -132,7 +145,11 @@ var _ = Describe("Model test", func() {
 			c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = InstallModel(tempdir, "foo", c, map[string]interface{}{"backend": "foo"}, func(string, string, string, float64) {}, true)
+			systemState, err := system.GetSystemState(
+				system.WithModelPath(tempdir),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = InstallModel(systemState, "foo", c, map[string]interface{}{"backend": "foo"}, func(string, string, string, float64) {}, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			for _, f := range []string{"cerebras", "cerebras-completion.tmpl", "cerebras-chat.tmpl", "foo.yaml"} {
@@ -158,7 +175,11 @@ var _ = Describe("Model test", func() {
 			c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = InstallModel(tempdir, "../../../foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
+			systemState, err := system.GetSystemState(
+				system.WithModelPath(tempdir),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = InstallModel(systemState, "../../../foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
 			Expect(err).To(HaveOccurred())
 		})
 	})