mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 02:18:50 -04:00
b19afb192a
* fix(backends): list backends runnable on worker nodes in distributed mode GET /backends/available filtered the gallery against the system state of the host serving the request. In a distributed deployment that host is the controller, which typically has no GPU, while the GPUs live on worker nodes. Any meta backend whose capabilities map lacks a "default" (or "cpu") key was therefore dropped from the listing entirely — longcat-video, vllm-omni, ltx-video, parakeet, edgetam and qwentts were invisible in the UI even though installing them by name on a GPU worker worked fine. Workers now report their own meta-backend capability at registration and the controller persists it on the node row. The controller cannot derive it: OS-dependent capabilities (metal, darwin-x86, nvidia-l4t) and the CUDA runtime refinements are only observable on the worker. Nodes registered before this field existed fall back to a coarse capability derived from their GPU vendor and VRAM. Backend discovery then evaluates compatibility as the union over healthy backend nodes, so a backend runnable on any node is offered while one no node can run stays hidden. Each remote capability is evaluated through a capability-pinned system state, otherwise a forced capability on the controller image (LOCALAI_FORCE_META_BACKEND_CAPABILITY or /run/localai/capability) would silently override every worker's verdict. With no registered nodes the listing is byte-for-byte what it was, so single-node deployments are unaffected. Also fixes the same-root-cause misclassification in /api/operations, which used the capability-filtered listing to decide whether an operation was a backend or a model install. A GPU-only backend installing on a worker is still a backend operation on the controller, so that lookup is now unfiltered. Assisted-by: Claude:claude-opus-4-8 golangci-lint Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(backends): union worker capabilities in backend discovery Implementation for the specs added in the previous commit, plus the two remaining discovery endpoints. Capability-filtered backend discovery evaluated compatibility against the system state of the host serving the request. In a distributed deployment that host is the controller, which typically has no GPU, while the GPUs live on worker nodes. Any meta backend whose capabilities map lacks a "default" (or "cpu") key was dropped entirely — longcat-video, vllm-omni, ltx-video, parakeet, edgetam and qwentts were invisible in the UI even though installing them by name on a GPU worker worked fine. Workers now report their own meta-backend capability at registration and the controller persists it on the node row. The controller cannot derive it: OS-dependent capabilities (metal, darwin-x86, nvidia-l4t) and the CUDA runtime refinements are only observable on the worker. Nodes registered before this field existed fall back to a coarse capability derived from their GPU vendor and VRAM. Discovery then evaluates compatibility as the union over healthy backend nodes, so a backend runnable on any node is offered while one no node can run stays hidden. Each remote capability is evaluated through a capability-pinned system state, otherwise a forced capability on the controller image (LOCALAI_FORCE_META_BACKEND_CAPABILITY or /run/localai/capability) would silently override every worker's verdict. With no registered nodes the listing is byte-for-byte what it was, so single-node deployments are unaffected. Four surfaces shared this root cause and are all routed through the same helper now: - GET /backends/available - GET /api/fine-tuning/backends - GET /api/quantization/backends - /api/operations backend-vs-model classification, which additionally had no reason to filter by capability at all: a GPU-only backend installing on a worker is still a backend operation on the controller, so that lookup is now unfiltered. Assisted-by: Claude:claude-opus-4-8 golangci-lint Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
42 lines
1.6 KiB
Go
42 lines
1.6 KiB
Go
package routes
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/labstack/echo/v4"
|
|
"github.com/mudler/LocalAI/core/application"
|
|
"github.com/mudler/LocalAI/core/config"
|
|
"github.com/mudler/LocalAI/core/http/endpoints/localai"
|
|
"github.com/mudler/LocalAI/core/services/quantization"
|
|
)
|
|
|
|
// RegisterQuantizationRoutes registers quantization API routes.
|
|
func RegisterQuantizationRoutes(e *echo.Echo, qService *quantization.QuantizationService, appConfig *config.ApplicationConfig, app *application.Application, quantizationMw echo.MiddlewareFunc) {
|
|
if qService == nil {
|
|
return
|
|
}
|
|
|
|
// Service readiness middleware
|
|
readyMw := func(next echo.HandlerFunc) echo.HandlerFunc {
|
|
return func(c echo.Context) error {
|
|
if qService == nil {
|
|
return c.JSON(http.StatusServiceUnavailable, map[string]string{
|
|
"error": "quantization service is not available",
|
|
})
|
|
}
|
|
return next(c)
|
|
}
|
|
}
|
|
|
|
q := e.Group("/api/quantization", readyMw, quantizationMw)
|
|
q.GET("/backends", localai.ListQuantizationBackendsEndpoint(appConfig, ClusterCapabilityProviderFor(app)))
|
|
q.POST("/jobs", localai.StartQuantizationJobEndpoint(qService))
|
|
q.GET("/jobs", localai.ListQuantizationJobsEndpoint(qService))
|
|
q.GET("/jobs/:id", localai.GetQuantizationJobEndpoint(qService))
|
|
q.POST("/jobs/:id/stop", localai.StopQuantizationJobEndpoint(qService))
|
|
q.DELETE("/jobs/:id", localai.DeleteQuantizationJobEndpoint(qService))
|
|
q.GET("/jobs/:id/progress", localai.QuantizationProgressEndpoint(qService))
|
|
q.POST("/jobs/:id/import", localai.ImportQuantizedModelEndpoint(qService))
|
|
q.GET("/jobs/:id/download", localai.DownloadQuantizedModelEndpoint(qService))
|
|
}
|