LocalAI/core/http/endpoints/localai/backend.go

package localai

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"

	"github.com/google/uuid"
	"github.com/labstack/echo/v4"
	"github.com/mudler/LocalAI/core/config"
	"github.com/mudler/LocalAI/core/gallery"
	"github.com/mudler/LocalAI/core/gallery/importers"
	"github.com/mudler/LocalAI/core/http/middleware"
	"github.com/mudler/LocalAI/core/schema"
	"github.com/mudler/LocalAI/core/services/galleryop"
	"github.com/mudler/LocalAI/pkg/system"
	"github.com/mudler/xlog"
)

// knownPrefOnlyBackends lists backends that have no dedicated importer
// and no importer-hosted drop-in entry, but users may still pick them
// via the preference-only path. Edit this slice to add new pref-only
// backends that should appear in the import form dropdown.
var knownPrefOnlyBackends = []schema.KnownBackend{
	// Text LLM
	// ds4: antirez/ds4 - single-model DeepSeek V4 Flash engine; auto-detected via DS4Importer
	{Name: "ds4", Modality: "text", AutoDetect: false, Description: "antirez/ds4 DeepSeek V4 Flash engine (auto-detected; pref-only fallback)"},
	// privacy-filter is now auto-detected via PrivacyFilterImporter (see
	// core/gallery/importers/privacy-filter.go); the importer registry entry
	// supersedes any pref-only line here, which the /backends/known merge would
	// dedupe away.
	{Name: "sglang", Modality: "text", AutoDetect: false, Description: "SGLang runtime (preference-only)"},
	{Name: "tinygrad", Modality: "text", AutoDetect: false, Description: "tinygrad runtime (preference-only)"},
	{Name: "trl", Modality: "text", AutoDetect: false, Description: "Transformers Reinforcement Learning (preference-only)"},
	{Name: "mlx-vlm", Modality: "text", AutoDetect: false, Description: "MLX vision-language models (preference-only)"},
	// ASR
	{Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"},
	{Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"},
	// nemo-speech-cpp serves four families from one backend, picked at load time
	// from the GGUF general.architecture key. Modality is a single string and the
	// import form chips on a fixed key set
	// (core/http/react-ui/src/components/ModalityChips.jsx), so "asr" is the one
	// it carries and the other three are named in the description rather than in
	// a key the UI would bucket as "other".
	// No importer: general.architecture lives inside the GGUF and cannot be read
	// from a remote HuggingFace repo, and the NMT family carries an ordinary LLM
	// architecture (qwen3), so even a local probe has no NeMo-specific string to
	// match on.
	{Name: "nemo-speech-cpp", Modality: "asr", AutoDetect: false, Description: "NVIDIA NeMo-Speech.cpp: Nemotron ASR (offline, streaming and live), Sortformer diarization, MagpieTTS speech synthesis and Riva-Translate translation, chosen from the model's GGUF architecture (preference-only)"},
	// TTS
	{Name: "mlx-audio", Modality: "tts", AutoDetect: false, Description: "MLX-Audio text-to-speech models (auto-detected; pref-only fallback)"},
	{Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"},
	{Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"},
	{Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"},
	{Name: "magpie-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Magpie TTS Multilingual C++ (preference-only)"},
	{Name: "omnivoice-cpp", Modality: "tts", AutoDetect: false, Description: "OmniVoice C++ TTS with voice cloning and voice design (preference-only)"},
	{Name: "faster-qwen3-tts", Modality: "tts", AutoDetect: false, Description: "Faster Qwen3 TTS (preference-only)"},
	{Name: "supertonic", Modality: "tts", AutoDetect: false, Description: "Supertonic multilingual ONNX TTS (preference-only)"},
	// audio-cpp spans far more than TTS: ASR, forced alignment, VAD, speaker
	// diarization, source separation, voice conversion and music generation.
	// KnownBackend.Modality is a single string and the import form only chips
	// on a fixed key set (core/http/react-ui/src/components/ModalityChips.jsx),
	// so the extra modalities live in the description rather than in an
	// invented modality key the UI would bucket as "other".
	// No importer: the only reliable signal, the audiocpp.model_spec.family
	// GGUF metadata key, is not readable from a remote HuggingFace repo, and
	// audio-cpp/audio.cpp-gguf hosts 30-odd families in one repository.
	{Name: "audio-cpp", Modality: "tts", AutoDetect: false, Description: "audio.cpp multi-family audio engine: TTS, voice cloning, ASR, alignment, VAD, diarization, separation, music generation (preference-only)"},
	// Detection
	{Name: "sam3-cpp", Modality: "detection", AutoDetect: false, Description: "SAM3 C++ object detection (preference-only)"},
	// Audio transform (audio-in / audio-out, optional reference signal)
	{Name: "localvqe", Modality: "audio-transform", AutoDetect: false, Description: "LocalVQE C++ joint AEC + noise suppression + dereverberation (preference-only)"},
}

// UpgradeInfoProvider is an interface for querying cached backend upgrade information.
type UpgradeInfoProvider interface {
	GetAvailableUpgrades() map[string]gallery.UpgradeInfo
	TriggerCheck()
}

type BackendEndpointService struct {
	galleries         []config.Gallery
	backendPath       string
	backendSystemPath string
	backendApplier    *galleryop.GalleryService
	upgradeChecker    UpgradeInfoProvider
}

type GalleryBackend struct {
	ID string `json:"id"`
	// Force reinstalls the backend even when it is already installed and
	// runnable. Off by default so apply stays idempotent for supervising
	// apps that ensure their backend on every boot.
	Force bool `json:"force"`
}

func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService, upgradeChecker UpgradeInfoProvider) BackendEndpointService {
	return BackendEndpointService{
		galleries:         galleries,
		backendPath:       systemState.Backend.BackendsPath,
		backendSystemPath: systemState.Backend.BackendsSystemPath,
		backendApplier:    backendApplier,
		upgradeChecker:    upgradeChecker,
	}
}

// GetOpStatusEndpoint returns the job status
// @Summary Returns the job status
// @Tags backends
// @Success 200 {object} galleryop.OpStatus "Response"
// @Router /backends/jobs/{uuid} [get]
func (mgs *BackendEndpointService) GetOpStatusEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		status := mgs.backendApplier.GetStatus(c.Param("uuid"))
		if status == nil {
			return fmt.Errorf("could not find any status for ID")
		}
		return c.JSON(200, status)
	}
}

// GetAllStatusEndpoint returns all the jobs status progress
// @Summary Returns all the jobs status progress
// @Tags backends
// @Success 200 {object} map[string]galleryop.OpStatus "Response"
// @Router /backends/jobs [get]
func (mgs *BackendEndpointService) GetAllStatusEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		return c.JSON(200, mgs.backendApplier.GetAllStatus())
	}
}

// ApplyBackendEndpoint installs a new backend to a LocalAI instance. The op is
// idempotent: an already-installed, runnable backend is left alone unless the
// request sets "force": true (explicit reinstall).
// @Summary Install backends to LocalAI.
// @Tags backends
// @Param request body GalleryBackend true "query params"
// @Success 200 {object} schema.BackendResponse "Response"
// @Router /backends/apply [post]
func (mgs *BackendEndpointService) ApplyBackendEndpoint(systemState *system.SystemState) echo.HandlerFunc {
	return func(c echo.Context) error {
		input := new(GalleryBackend)
		// Get input data from the request body
		if err := c.Bind(input); err != nil {
			return err
		}

		// In distributed mode, refuse to fan out a hardware-specific build to
		// every node — a CPU build landing on a GPU cluster is almost always
		// wrong, and the silent footgun is exactly what this guard exists for.
		// Auto-resolving (meta) backends are fine because each node picks its
		// own variant. Tooling can recover by hitting
		// POST /api/nodes/{id}/backends/install per target node.
		if mgs.backendApplier.BackendManager().IsDistributed() && input.ID != "" {
			if guard := concreteFanOutGuard(c, mgs.galleries, systemState, input.ID); guard != nil {
				return guard
			}
		}

		uuid, err := uuid.NewUUID()
		if err != nil {
			return err
		}
		mgs.backendApplier.EnqueueBackendOp(galleryop.ManagementOp[gallery.GalleryBackend, any]{
			ID:                 uuid.String(),
			GalleryElementName: input.ID,
			Galleries:          mgs.galleries,
			Force:              input.Force,
		})

		return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())})
	}
}

// concreteFanOutGuard returns a 409 response if the requested backend is a
// hardware-specific build (not auto-resolving / meta) and we are in
// distributed mode. It looks up the backend in the configured galleries; if
// the lookup itself fails (gallery unreachable, name not found), the guard
// stays out of the way and lets the install enqueue normally — a missing
// name will surface from the worker as a clearer error than the guard could
// produce here. The response body deliberately speaks human, with `code` and
// `meta_alternative` as the programmatic contract for tooling.
func concreteFanOutGuard(c echo.Context, galleries []config.Gallery, systemState *system.SystemState, backendID string) error {
	// Use the unfiltered listing because in distributed mode the frontend's
	// hardware is irrelevant — the install targets workers, not us — and the
	// filtered list would hide variants that don't match the frontend host
	// (e.g. a CUDA build on a CPU-only frontend), preventing the guard from
	// firing for exactly the cases it's meant to protect against.
	available, err := gallery.AvailableBackendsUnfiltered(galleries, systemState)
	if err != nil {
		return nil
	}
	requested := available.FindByName(backendID)
	if requested == nil || requested.IsMeta() {
		return nil
	}

	// Try to find an auto-resolving (meta) backend that has this concrete
	// variant in its CapabilitiesMap, so we can suggest it as a one-shot
	// alternative. Optional — empty string is fine if no parent exists.
	metaAlternative := ""
	for _, b := range available {
		if !b.IsMeta() {
			continue
		}
		for _, concrete := range b.CapabilitiesMap {
			if concrete == backendID {
				metaAlternative = b.Name
				break
			}
		}
		if metaAlternative != "" {
			break
		}
	}

	msg := fmt.Sprintf(
		"Backend %q is a hardware-specific build and won't run correctly on every node in this cluster. In distributed mode, install it on specific nodes:\n\n  POST /api/nodes/{node_id}/backends/install\n  {\"backend\": %q}",
		backendID, backendID,
	)
	if metaAlternative != "" {
		msg += fmt.Sprintf(
			"\n\nTo install across all nodes, use the auto-resolving backend %q — each node picks its own variant based on its hardware.",
			metaAlternative,
		)
	}

	return c.JSON(409, map[string]any{
		"error":            msg,
		"code":             "concrete_backend_requires_target",
		"meta_alternative": metaAlternative,
	})
}

// DeleteBackendEndpoint lets delete backends from a LocalAI instance
// @Summary delete backends from LocalAI.
// @Tags backends
// @Param name	path string	true	"Backend name"
// @Success 200 {object} schema.BackendResponse "Response"
// @Router /backends/delete/{name} [post]
func (mgs *BackendEndpointService) DeleteBackendEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		backendName := c.Param("name")

		uuid, err := uuid.NewUUID()
		if err != nil {
			return err
		}

		// The op carries the same ID the caller is handed back: without it the
		// deletion ran under an empty ID and StatusURL pointed at a job that
		// could never have a status.
		mgs.backendApplier.EnqueueBackendOp(galleryop.ManagementOp[gallery.GalleryBackend, any]{
			ID:                 uuid.String(),
			Delete:             true,
			GalleryElementName: backendName,
			Galleries:          mgs.galleries,
		})

		return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())})
	}
}

// ListBackendsEndpoint list the available backends configured in LocalAI
// @Summary List all Backends
// @Tags backends
// @Success 200 {object} []gallery.GalleryBackend "Response"
// @Router /backends [get]
func (mgs *BackendEndpointService) ListBackendsEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		backends, err := mgs.backendApplier.ListBackends()
		if err != nil {
			return err
		}
		return c.JSON(200, backends.GetAll())
	}
}

// ListBackendGalleriesEndpoint lists the available backend galleries configured in LocalAI.
// @Summary List all Galleries
// @Tags backends
// @Success 200 {object} []config.Gallery "Response"
// @Router /backends/galleries [get]
// NOTE: This is different (and much simpler!) than above! This JUST lists the model galleries that have been loaded, not their contents!
func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		xlog.Debug("Listing backend galleries", "galleries", mgs.galleries)
		dat, err := json.Marshal(mgs.galleries)
		if err != nil {
			return err
		}
		return c.Blob(200, "application/json", dat)
	}
}

// GetUpgradesEndpoint returns the cached backend upgrade information
// @Summary Get available backend upgrades
// @Tags backends
// @Success 200 {object} map[string]gallery.UpgradeInfo "Response"
// @Router /backends/upgrades [get]
func (mgs *BackendEndpointService) GetUpgradesEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		if mgs.upgradeChecker == nil {
			return c.JSON(200, map[string]gallery.UpgradeInfo{})
		}
		return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades())
	}
}

// CheckUpgradesEndpoint forces an immediate upgrade check
// @Summary Force backend upgrade check
// @Tags backends
// @Success 200 {object} map[string]gallery.UpgradeInfo "Response"
// @Router /backends/upgrades/check [post]
func (mgs *BackendEndpointService) CheckUpgradesEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		if mgs.upgradeChecker == nil {
			return c.JSON(200, map[string]gallery.UpgradeInfo{})
		}
		mgs.upgradeChecker.TriggerCheck()
		// Return current cached results (the triggered check runs async)
		return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades())
	}
}

// UpgradeBackendEndpoint triggers an upgrade for a specific backend
// @Summary Upgrade a backend
// @Tags backends
// @Param name path string true "Backend name"
// @Success 200 {object} schema.BackendResponse "Response"
// @Router /backends/upgrade/{name} [post]
func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc {
	return func(c echo.Context) error {
		backendName := c.Param("name")

		uuid, err := uuid.NewUUID()
		if err != nil {
			return err
		}

		mgs.backendApplier.EnqueueBackendOp(galleryop.ManagementOp[gallery.GalleryBackend, any]{
			ID:                 uuid.String(),
			GalleryElementName: backendName,
			Galleries:          mgs.galleries,
			Upgrade:            true,
		})

		return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())})
	}
}

// ClusterCapabilityProvider reports the meta-backend capabilities available
// somewhere in the cluster. It is nil in single-node deployments, where the
// local system state is the only thing worth filtering against.
type ClusterCapabilityProvider func(ctx context.Context) ([]string, error)

// ResolveClusterCapabilities reads the capabilities present in the cluster,
// degrading to the local-only listing on error.
//
// Every capability-filtered discovery endpoint shares this: on a distributed
// controller the GPUs live on the workers, so filtering against the local
// (usually GPU-less) host hides GPU-only backends the cluster can actually
// run. A registry hiccup must never blank the catalog, so a failure falls back
// to the pre-existing local-only behavior rather than erroring the request.
func ResolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
	if provider == nil {
		return nil
	}
	capabilities, err := provider(ctx)
	if err != nil {
		xlog.Warn("Could not read cluster capabilities, listing backends for the local system only", "error", err)
		return nil
	}
	return capabilities
}

// ClusterInstalledProvider reports the backends installed somewhere in the
// cluster. It is nil in single-node deployments, where the local filesystem is
// the only install state that exists.
type ClusterInstalledProvider func(ctx context.Context) ([]string, error)

// resolveClusterInstalled reads the backends installed across the cluster,
// degrading to the local-only view on error.
//
// Every discovery endpoint that filters on installed-state shares this: a
// backend lives on the worker node that runs it, so the controller's own
// filesystem reports it missing and the endpoint hides it. A nil set leaves
// the local filesystem as the only source, so single-node listings are
// untouched, and a registry hiccup degrades to that same listing rather than
// erroring the request.
func resolveClusterInstalled(ctx context.Context, provider ClusterInstalledProvider) map[string]struct{} {
	if provider == nil {
		return nil
	}
	names, err := provider(ctx)
	if err != nil {
		xlog.Warn("Could not read cluster backend install state, reporting the local system only", "error", err)
		return nil
	}
	installed := make(map[string]struct{}, len(names))
	for _, name := range names {
		installed[name] = struct{}{}
	}
	return installed
}

// installedInCluster reports whether a backend is installed on the host serving
// the listing or on any node of the cluster.
func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[string]struct{}) bool {
	if backend.Installed {
		return true
	}
	_, ok := clusterInstalled[backend.Name]
	return ok
}

// ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI
// @Summary List all available Backends
// @Tags backends
// @Success 200 {object} []gallery.GalleryBackend "Response"
// @Router /backends/available [get]
func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
	return func(c echo.Context) error {
		capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)

		backends, err := gallery.AvailableBackendsForCapabilities(mgs.galleries, systemState, capabilities)
		if err != nil {
			return err
		}

		installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
		for _, b := range backends {
			b.SetInstalled(installedInCluster(b, installed))
		}

		return c.JSON(200, backends)
	}
}

// ListKnownBackendsEndpoint returns every backend the import system is
// aware of, regardless of install state or host compatibility. This is
// the source of truth for the import form dropdown — users may pick a
// backend that is not yet installed so LocalAI can auto-install it.
// @Summary List all known Backends (importer registry + curated pref-only + installed-on-disk)
// @Tags backends
// @Success 200 {object} []schema.KnownBackend "Response"
// @Router /backends/known [get]
func (mgs *BackendEndpointService) ListKnownBackendsEndpoint(systemState *system.SystemState) echo.HandlerFunc {
	return func(c echo.Context) error {
		// byName dedupes entries while preserving "importer wins over
		// pref-only" priority. Insertion order: importers → drop-ins →
		// pref-only → installed-on-disk.
		byName := make(map[string]schema.KnownBackend)

		for _, imp := range importers.Registry() {
			byName[imp.Name()] = schema.KnownBackend{
				Name:       imp.Name(),
				Modality:   imp.Modality(),
				AutoDetect: imp.AutoDetects(),
			}

			if host, ok := imp.(importers.AdditionalBackendsProvider); ok {
				for _, extra := range host.AdditionalBackends() {
					if _, exists := byName[extra.Name]; exists {
						continue
					}
					byName[extra.Name] = schema.KnownBackend{
						Name:        extra.Name,
						Modality:    extra.Modality,
						AutoDetect:  false,
						Description: extra.Description,
					}
				}
			}
		}

		for _, pref := range knownPrefOnlyBackends {
			if _, exists := byName[pref.Name]; exists {
				continue
			}
			byName[pref.Name] = pref
		}

		// Surface backends installed on this host and flag them as such.
		// Importer/pref-only entries that are also on disk get Installed=true
		// while keeping their metadata. System-only backends join the map
		// with empty Modality (we can't classify them) and AutoDetect=false
		// because they require an explicit preference.
		if systemState != nil {
			installed, err := gallery.ListSystemBackends(systemState)
			if err != nil {
				xlog.Debug("ListKnownBackendsEndpoint: failed to list installed backends", "error", err)
			} else {
				for name := range installed {
					if entry, exists := byName[name]; exists {
						entry.Installed = true
						byName[name] = entry
						continue
					}
					byName[name] = schema.KnownBackend{
						Name:       name,
						Modality:   "",
						AutoDetect: false,
						Installed:  true,
					}
				}
			}
		}

		out := make([]schema.KnownBackend, 0, len(byName))
		for _, b := range byName {
			out = append(out, b)
		}
		sort.SliceStable(out, func(i, j int) bool {
			if out[i].Modality != out[j].Modality {
				return out[i].Modality < out[j].Modality
			}
			return out[i].Name < out[j].Name
		})

		return c.JSON(200, out)
	}
}