fix(distributed): backend discovery hid worker-installed backends behind the controller's filesystem (#10967)

fix(distributed): backend discovery hid worker-installed backends

Backend discovery endpoints filter on installed-state, which on a
distributed controller derives from the controller's own filesystem. A
backend lives on the worker node that runs it, so every backend an admin
installed on a GPU worker read as "not installed" and vanished from the
listing. #10947 fixed the sibling capability filter on the same endpoints,
so a fine-tuning-capable GPU worker now made the backend listable while
the installed-state filter still dropped it: the dropdown stayed empty.

The controller cannot derive this locally, but it already aggregates the
per-node view that GET /backends renders, so discovery reuses the active
BackendManager rather than growing a second path. Three surfaces shared the
root cause and route through the same helper now:

  - GET /backends/available (Installed is now cluster-wide)
  - GET /api/fine-tuning/backends
  - GET /api/quantization/backends

The response stays a boolean rather than an installed-on-N-of-M count:
per-node install state is already served by GET /backends nodes[], and
per-node control by POST /api/nodes/:id/backends/install, so a summary is
all these dropdowns need.

A nil provider (single-node) leaves the local filesystem as the only source
and reproduces today's listing exactly, and a registry error degrades to
that same listing instead of blanking the catalog.


Assisted-by: Claude:claude-opus-4-8 golangci-lint

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

core/http/endpoints/localai/backend.go

@@ -349,12 +349,52 @@ func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityP
 	return capabilities
 }
 
+// ClusterInstalledProvider reports the backends installed somewhere in the
+// cluster. It is nil in single-node deployments, where the local filesystem is
+// the only install state that exists.
+type ClusterInstalledProvider func(ctx context.Context) ([]string, error)
+
+// resolveClusterInstalled reads the backends installed across the cluster,
+// degrading to the local-only view on error.
+//
+// Every discovery endpoint that filters on installed-state shares this: a
+// backend lives on the worker node that runs it, so the controller's own
+// filesystem reports it missing and the endpoint hides it. A nil set leaves
+// the local filesystem as the only source, so single-node listings are
+// untouched, and a registry hiccup degrades to that same listing rather than
+// erroring the request.
+func resolveClusterInstalled(ctx context.Context, provider ClusterInstalledProvider) map[string]struct{} {
+	if provider == nil {
+		return nil
+	}
+	names, err := provider(ctx)
+	if err != nil {
+		xlog.Warn("Could not read cluster backend install state, reporting the local system only", "error", err)
+		return nil
+	}
+	installed := make(map[string]struct{}, len(names))
+	for _, name := range names {
+		installed[name] = struct{}{}
+	}
+	return installed
+}
+
+// installedInCluster reports whether a backend is installed on the host serving
+// the listing or on any node of the cluster.
+func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[string]struct{}) bool {
+	if backend.Installed {
+		return true
+	}
+	_, ok := clusterInstalled[backend.Name]
+	return ok
+}
+
 // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI
 // @Summary List all available Backends
 // @Tags backends
 // @Success 200 {object} []gallery.GalleryBackend "Response"
 // @Router /backends/available [get]
-func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc {
+func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
 
@@ -362,6 +402,12 @@ func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *sy
 		if err != nil {
 			return err
 		}
+
+		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
+		for _, b := range backends {
+			b.SetInstalled(installedInCluster(b, installed))
+		}
+
 		return c.JSON(200, backends)
 	}
 }

core/http/endpoints/localai/backend_available_cluster_test.go

@@ -66,7 +66,7 @@ var _ = Describe("ListAvailableBackendsEndpoint cluster capabilities", func() {
 
 	listNames := func(provider ClusterCapabilityProvider) []string {
 		svc := CreateBackendEndpointService(galleries, systemState, nil, nil)
-		app.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, provider))
+		app.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, provider, nil))
 
 		req := httptest.NewRequest(http.MethodGet, "/backends/available", nil)
 		rec := httptest.NewRecorder()

core/http/endpoints/localai/discovery_cluster_capabilities_test.go

@@ -100,26 +100,26 @@ var _ = Describe("Tagged backend discovery with cluster capabilities", func() {
 
 	Describe("fine-tune backends", func() {
 		It("hides GPU-only backends with no cluster provider", func() {
-			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil))
+			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil, nil))
 			Expect(names).To(ContainElement("cpu-trainer"))
 			Expect(names).NotTo(ContainElement("gpu-trainer"))
 		})
 
 		It("lists GPU-only backends a worker node can run", func() {
-			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker))
+			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, nil))
 			Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer"))
 		})
 	})
 
 	Describe("quantization backends", func() {
 		It("hides GPU-only backends with no cluster provider", func() {
-			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil))
+			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil, nil))
 			Expect(names).To(ContainElement("cpu-trainer"))
 			Expect(names).NotTo(ContainElement("gpu-trainer"))
 		})
 
 		It("lists GPU-only backends a worker node can run", func() {
-			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker))
+			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker, nil))
 			Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer"))
 		})
 	})

core/http/endpoints/localai/discovery_cluster_installed_test.go

@@ -0,0 +1,174 @@
+package localai_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	. "github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+// Installed-state on a distributed controller is derived from the controller's
+// own filesystem, where a backend installed on a GPU worker does not exist.
+// The capability fix (#10947) made those backends *listable*; every surface
+// that filters on installed-state still dropped them, so an admin with a
+// fine-tuning-capable GPU worker got an empty dropdown.
+var _ = Describe("Backend discovery with cluster install state", func() {
+	var (
+		appCfg      *config.ApplicationConfig
+		systemState *system.SystemState
+		galleries   []config.Gallery
+		tmpDir      string
+	)
+
+	BeforeEach(func() {
+		var err error
+		tmpDir, err = os.MkdirTemp("", "discovery-cluster-installed-*")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() {
+			Expect(os.RemoveAll(tmpDir)).To(Succeed())
+		})
+
+		// cpu-trainer sits on the controller's disk; gpu-trainer exists only on
+		// a worker, so nothing on this filesystem can prove it is installed.
+		writeFakeSystemBackend(tmpDir, "cpu-trainer")
+
+		galleryPath := filepath.Join(tmpDir, "gallery.yaml")
+		data, err := yaml.Marshal([]gallery.GalleryBackend{
+			{
+				Metadata: gallery.Metadata{
+					Name: "gpu-trainer",
+					Tags: []string{"fine-tuning", "quantization"},
+				},
+				CapabilitiesMap: map[string]string{"nvidia-cuda-13": "gpu-trainer-cuda-13"},
+			},
+			{
+				Metadata: gallery.Metadata{
+					Name: "cpu-trainer",
+					Tags: []string{"fine-tuning", "quantization"},
+				},
+				CapabilitiesMap: map[string]string{"default": "cpu-trainer-cpu"},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, data, 0o644)).To(Succeed())
+
+		galleries = []config.Gallery{{Name: "test-gallery", URL: "file://" + galleryPath}}
+		// A GPU-less controller pod: capability "default".
+		systemState = system.NewCapabilityState("default",
+			system.WithBackendPath(tmpDir), system.WithBackendSystemPath(tmpDir))
+		appCfg = &config.ApplicationConfig{
+			BackendGalleries: galleries,
+			SystemState:      systemState,
+		}
+	})
+
+	nvidiaWorker := func(ctx context.Context) ([]string, error) {
+		return []string{"nvidia-cuda-13"}, nil
+	}
+	installedOnWorker := func(ctx context.Context) ([]string, error) {
+		return []string{"gpu-trainer"}, nil
+	}
+	registryDown := func(ctx context.Context) ([]string, error) {
+		return nil, errors.New("registry unavailable")
+	}
+
+	// listNames drives handler over its route and returns the backend names.
+	listNames := func(path string, handler echo.HandlerFunc) []string {
+		e := echo.New()
+		e.GET(path, handler)
+
+		req := httptest.NewRequest(http.MethodGet, path, nil)
+		rec := httptest.NewRecorder()
+		e.ServeHTTP(rec, req)
+		Expect(rec.Code).To(Equal(http.StatusOK))
+
+		var backends []struct {
+			Name string `json:"name"`
+		}
+		Expect(json.Unmarshal(rec.Body.Bytes(), &backends)).To(Succeed())
+
+		names := []string{}
+		for _, b := range backends {
+			names = append(names, b.Name)
+		}
+		return names
+	}
+
+	Describe("fine-tune backends", func() {
+		It("lists a backend installed only on a worker node", func() {
+			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, installedOnWorker))
+			Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer"))
+		})
+
+		It("keeps single-node listings unchanged with no provider", func() {
+			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nil, nil))
+			Expect(names).To(ContainElement("cpu-trainer"))
+			Expect(names).NotTo(ContainElement("gpu-trainer"))
+		})
+
+		It("degrades to the local listing when the registry errors", func() {
+			names := listNames("/backends", ListFineTuneBackendsEndpoint(appCfg, nvidiaWorker, registryDown))
+			Expect(names).To(ContainElement("cpu-trainer"))
+			Expect(names).NotTo(ContainElement("gpu-trainer"))
+		})
+	})
+
+	Describe("quantization backends", func() {
+		It("lists a backend installed only on a worker node", func() {
+			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nvidiaWorker, installedOnWorker))
+			Expect(names).To(ContainElements("cpu-trainer", "gpu-trainer"))
+		})
+
+		It("keeps single-node listings unchanged with no provider", func() {
+			names := listNames("/backends", ListQuantizationBackendsEndpoint(appCfg, nil, nil))
+			Expect(names).To(ContainElement("cpu-trainer"))
+			Expect(names).NotTo(ContainElement("gpu-trainer"))
+		})
+	})
+
+	Describe("GET /backends/available", func() {
+		installedFlags := func(capabilities ClusterCapabilityProvider, installed ClusterInstalledProvider) map[string]bool {
+			e := echo.New()
+			svc := CreateBackendEndpointService(galleries, systemState, nil, nil)
+			e.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, capabilities, installed))
+
+			req := httptest.NewRequest(http.MethodGet, "/backends/available", nil)
+			rec := httptest.NewRecorder()
+			e.ServeHTTP(rec, req)
+			Expect(rec.Code).To(Equal(http.StatusOK))
+
+			var backends []gallery.GalleryBackend
+			Expect(json.Unmarshal(rec.Body.Bytes(), &backends)).To(Succeed())
+
+			flags := map[string]bool{}
+			for _, b := range backends {
+				flags[b.Name] = b.Installed
+			}
+			return flags
+		}
+
+		It("reports a worker-installed backend as installed", func() {
+			flags := installedFlags(nvidiaWorker, installedOnWorker)
+			Expect(flags).To(HaveKeyWithValue("gpu-trainer", true))
+			Expect(flags).To(HaveKeyWithValue("cpu-trainer", true))
+		})
+
+		It("keeps single-node install state unchanged with no provider", func() {
+			flags := installedFlags(nvidiaWorker, nil)
+			Expect(flags).To(HaveKeyWithValue("gpu-trainer", false))
+			Expect(flags).To(HaveKeyWithValue("cpu-trainer", true))
+		})
+	})
+})

core/http/endpoints/localai/finetune.go

@@ -274,9 +274,10 @@ func DownloadExportedModelEndpoint(ftService *finetune.FineTuneService) echo.Han
 }
 
 // ListFineTuneBackendsEndpoint returns installed backends tagged with "fine-tuning".
-func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc {
+func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
 		backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
 		if err != nil {
 			return c.JSON(http.StatusInternalServerError, map[string]string{
@@ -292,7 +293,7 @@ func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCa
 
 		var result []backendInfo
 		for _, b := range backends {
-			if !b.Installed {
+			if !installedInCluster(b, installed) {
 				continue
 			}
 			hasTag := false

core/http/endpoints/localai/quantization.go

@@ -193,9 +193,10 @@ func DownloadQuantizedModelEndpoint(qService *quantization.QuantizationService)
 }
 
 // ListQuantizationBackendsEndpoint returns installed backends tagged with "quantization".
-func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc {
+func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
 		backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
 		if err != nil {
 			return c.JSON(http.StatusInternalServerError, map[string]string{
@@ -211,7 +212,7 @@ func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clust
 
 		var result []backendInfo
 		for _, b := range backends {
-			if !b.Installed {
+			if !installedInCluster(b, installed) {
 				continue
 			}
 			hasTag := false

core/http/routes/cluster_capabilities.go

@@ -1,6 +1,8 @@
 package routes
 
 import (
+	"context"
+
 	"github.com/mudler/LocalAI/core/application"
 	"github.com/mudler/LocalAI/core/http/endpoints/localai"
 )
@@ -18,3 +20,33 @@ func ClusterCapabilityProviderFor(app *application.Application) localai.ClusterC
 	}
 	return app.Distributed().Registry.HealthyBackendCapabilities
 }
+
+// ClusterInstalledProviderFor returns the install-state source backing every
+// backend discovery endpoint that filters on installed backends, or nil in
+// single-node mode.
+//
+// A nil provider leaves those endpoints reading the local filesystem exactly as
+// they always have. In distributed mode a backend lives on the worker that runs
+// it, so the controller's own disk cannot answer the question; the active
+// BackendManager already aggregates the per-node view that GET /backends
+// renders, and discovery reuses it rather than growing a second path.
+func ClusterInstalledProviderFor(app *application.Application) localai.ClusterInstalledProvider {
+	if app == nil || !app.IsDistributed() || app.GalleryService() == nil {
+		return nil
+	}
+	return func(ctx context.Context) ([]string, error) {
+		manager := app.GalleryService().BackendManager()
+		if manager == nil {
+			return nil, nil
+		}
+		backends, err := manager.ListBackends()
+		if err != nil {
+			return nil, err
+		}
+		names := make([]string, 0, len(backends))
+		for name := range backends {
+			names = append(names, name)
+		}
+		return names, nil
+	}
+}

core/http/routes/finetuning.go

@@ -29,7 +29,7 @@ func RegisterFineTuningRoutes(e *echo.Echo, ftService *finetune.FineTuneService,
 	}
 
 	ft := e.Group("/api/fine-tuning", readyMw, fineTuningMw)
-	ft.GET("/backends", localai.ListFineTuneBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app)))
+	ft.GET("/backends", localai.ListFineTuneBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app)))
 	ft.POST("/jobs", localai.StartFineTuneJobEndpoint(ftService))
 	ft.GET("/jobs", localai.ListFineTuneJobsEndpoint(ftService))
 	ft.GET("/jobs/:id", localai.GetFineTuneJobEndpoint(ftService))

core/http/routes/localai.go

@@ -64,7 +64,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 		router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(appConfig.SystemState), adminMiddleware)
 		router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware)
 		router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware)
-		router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState, ClusterCapabilityProviderFor(app)), adminMiddleware)
+		router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app)), adminMiddleware)
 		router.GET("/backends/known", backendGalleryEndpointService.ListKnownBackendsEndpoint(appConfig.SystemState), adminMiddleware)
 		router.GET("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint(), adminMiddleware)
 		router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware)

core/http/routes/quantization.go

@@ -29,7 +29,7 @@ func RegisterQuantizationRoutes(e *echo.Echo, qService *quantization.Quantizatio
 	}
 
 	q := e.Group("/api/quantization", readyMw, quantizationMw)
-	q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app)))
+	q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app), ClusterInstalledProviderFor(app)))
 	q.POST("/jobs", localai.StartQuantizationJobEndpoint(qService))
 	q.GET("/jobs", localai.ListQuantizationJobsEndpoint(qService))
 	q.GET("/jobs/:id", localai.GetQuantizationJobEndpoint(qService))