fix(config): cap auto-derived context to fit VRAM

When a model is imported without an explicit context_size, the GGUF
importer defaulted the model's context to its full trained window
(n_ctx_train). For long-context models (128k / 256k / 1M) that KV cache
cannot fit a consumer GPU, so the backend aborts on load (exitCode=-1)
even though the model file is perfectly fine. Reproduced live:
gemma-4-26b-a4b-it-qat-q4_0 defaulted to context=262144 and
qwythos-9b-claude-mythos-5-1m to 1048576, both aborting on a 20 GB card.

Instead of chasing the trained max, auto-derive a conservative default:
min(trainedMax, DefaultAutoContextSize=8192). A small model keeps its
trained window; a long-context model caps at 8k and users opt into more
via context_size. This cap applies always, including CPU / unknown-VRAM
hosts, so it never regresses those paths.

Per-device VRAM is used only as a DOWNWARD safety: when a per-device
ceiling is detected (xsysinfo.MinPerGPUVRAM) and even the 8k cap would
not fit it with headroom, step down through candidate contexts to the
largest that fits, floored at DefaultContextSize. When VRAM is unknown
(0) or no GPU is detected we do NOT clamp — the bug is GPU OOM and the
8k cap is already safe, so detection gaps must not shrink the window.

The footprint estimate reuses gpustack/gguf-parser-go's
EstimateLLaMACppRun at a given context with all layers offloaded, taking
the per-device NonUMA VRAM figure. The estimate and VRAM detection are
package vars so tests inject deterministic values. Explicit context_size
always wins (guessGGUFFromFile only acts when it is nil).

Assisted-by: Claude:claude-opus-4-8 [golangci-lint go-test]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=fdb1db877c526ec90f668eca1b858da5dba85560
+LLAMA_VERSION?=2da668617612d2df773f966e3b0ee22dc2beef7b
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/run.sh

@@ -36,6 +36,12 @@ else
 	if [ -d "$CURDIR/lib/rocblas/library" ]; then
 		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
 	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/cpp/turboquant/run.sh

@@ -34,6 +34,12 @@ else
 	if [ -d "$CURDIR/lib/rocblas/library" ]; then
 		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
 	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/go/acestep-cpp/CMakeLists.txt

@@ -25,7 +25,7 @@ target_include_directories(goacestepcpp PRIVATE ${ACESTEP_DIR}/src ${ACESTEP_DIR
 target_include_directories(goacestepcpp SYSTEM PRIVATE ${ACESTEP_DIR}/ggml/include)
 
 # Link GPU backends if available (mirrors link_ggml_backends macro)
-foreach(backend blas cuda metal vulkan)
+foreach(backend blas cuda hip metal vulkan)
     if(TARGET ggml-${backend})
         target_link_libraries(goacestepcpp PRIVATE ggml-${backend})
         string(TOUPPER ${backend} BACKEND_UPPER)

backend/go/acestep-cpp/Makefile

@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=9a26976a8c8cf5af0afcdd04463cf8ba91e96a54
+CRISPASR_VERSION?=09df654e304947f7521e1f52992ceacccf03c300
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/omnivoice-cpp/CMakeLists.txt

@@ -30,7 +30,7 @@ target_include_directories(gomnivoicecpp PRIVATE ${OMNIVOICE_DIR}/src)
 target_include_directories(gomnivoicecpp SYSTEM PRIVATE ${OMNIVOICE_DIR}/ggml/include)
 
 # Link GPU backends if the upstream ggml created them.
-foreach(backend blas cuda metal vulkan sycl)
+foreach(backend blas cuda hip metal vulkan sycl)
     if(TARGET ggml-${backend})
         target_link_libraries(gomnivoicecpp PRIVATE ggml-${backend})
         if(backend STREQUAL "cuda")

backend/go/omnivoice-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # omnivoice.cpp version
 OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp
-OMNIVOICE_VERSION?=0f37401bebe9b20c0160a888e592108fc1d17607
+OMNIVOICE_VERSION?=daedb763fd442e0916eb130a479fdd74947291c0
 SO_TARGET?=libgomnivoicecpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/qwen3-tts-cpp/CMakeLists.txt

@@ -30,7 +30,7 @@ target_include_directories(goqwen3ttscpp PRIVATE ${QWENTTS_DIR}/src)
 target_include_directories(goqwen3ttscpp SYSTEM PRIVATE ${QWENTTS_DIR}/ggml/include)
 
 # Link GPU backends if the upstream ggml created them.
-foreach(backend blas cuda metal vulkan sycl)
+foreach(backend blas cuda hip metal vulkan sycl)
     if(TARGET ggml-${backend})
         target_link_libraries(goqwen3ttscpp PRIVATE ggml-${backend})
         if(backend STREQUAL "cuda")

backend/go/qwen3-tts-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # qwentts.cpp version
 QWEN3TTS_REPO?=https://github.com/ServeurpersoCom/qwentts.cpp
-QWEN3TTS_CPP_VERSION?=9dbe7ea26a01b30fccb117ae5e86807c1dc23d42
+QWEN3TTS_CPP_VERSION?=73fe0c67bbf0898ba2999535e0680a02a7f8537d
 SO_TARGET?=libgoqwen3ttscpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
@@ -24,7 +24,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/go/vibevoice-cpp/CMakeLists.txt

@@ -50,7 +50,7 @@ target_include_directories(govibevoicecpp SYSTEM PRIVATE ${VIBEVOICE_DIR}/third_
 # Link GPU backends if available — vibevoice's own CMake already links
 # these to the libvibevoice STATIC library, but we re-link them on the
 # MODULE so resolved symbols include all backend kernels.
-foreach(backend blas cuda metal vulkan)
+foreach(backend blas cuda hip metal vulkan)
     if(TARGET ggml-${backend})
         target_link_libraries(govibevoicecpp PRIVATE ggml-${backend})
         string(TOUPPER ${backend} BACKEND_UPPER)

backend/go/vibevoice-cpp/Makefile

@@ -29,7 +29,14 @@ else ifeq ($(BUILD_TYPE),openblas)
 else ifeq ($(BUILD_TYPE),clblas)
 	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
 else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DVIBEVOICE_GGML_HIPBLAS=ON
+	# This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream),
+	# so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666).
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON -DVIBEVOICE_GGML_VULKAN=ON
 else ifeq ($(OS),Darwin)

backend/python/vllm/install.sh

@@ -119,7 +119,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260701212152"
+    VLLM_METAL_VERSION="v0.3.0.dev20260704102955"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
     # against -- it declares it in its own installer as `vllm_v=`. Derive it from

core/backend/llm.go

@@ -47,6 +47,28 @@ func needsThinkingProbe(c *config.ModelConfig) bool {
 			c.ReasoningConfig.DisableReasoningTagPrefill == nil)
 }
 
+// persistProbedReasoning writes the post-probe reasoning slots (and media
+// marker) from probed back into the loader's persisted config for modelName,
+// skipping any reasoning slot the probe was not actually allowed to fill.
+// persistDisableReasoning/persistDisableTagPrefill must be snapshotted from
+// probed's reasoning slots *before* the probe ran: a slot that already
+// carried a value at that point was populated by request-time
+// ApplyReasoningEffort, not by backend detection, and persisting it would
+// masquerade as an operator's explicit reasoning.disable (see #10622).
+func persistProbedReasoning(cl *config.ModelConfigLoader, modelName string, probed *config.ModelConfig, persistDisableReasoning, persistDisableTagPrefill bool) {
+	cl.UpdateModelConfig(modelName, func(cfg *config.ModelConfig) {
+		if persistDisableReasoning {
+			cfg.ReasoningConfig.DisableReasoning = probed.ReasoningConfig.DisableReasoning
+		}
+		if persistDisableTagPrefill {
+			cfg.ReasoningConfig.DisableReasoningTagPrefill = probed.ReasoningConfig.DisableReasoningTagPrefill
+		}
+		if probed.MediaMarker != "" {
+			cfg.MediaMarker = probed.MediaMarker
+		}
+	})
+}
+
 // HasChatDeltaContent returns true if any chat delta carries content or reasoning text.
 // Used to decide whether to prefer C++ autoparser deltas over Go-side tag extraction.
 func (t TokenUsage) HasChatDeltaContent() bool {
@@ -127,15 +149,19 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 	needsMarkerProbe := c.MediaMarker == ""
 	if shouldProbeThinking || needsMarkerProbe {
 		modelOpts := grpcModelOpts(*c, o.SystemState.Model.ModelsPath)
+		// DetectThinkingSupportFromBackend only fills reasoning slots that are
+		// still nil, so a slot that already carries a value here was populated by
+		// request-time ApplyReasoningEffort (e.g. a `reasoning_effort: none`
+		// default), not by backend detection. Persisting such a request-scoped
+		// value would masquerade as an operator's explicit reasoning.disable and
+		// permanently defeat future per-request reasoning_effort overrides
+		// (see #10622). Only persist the slots the probe is actually allowed to
+		// fill.
+		persistDisableReasoning := c.ReasoningConfig.DisableReasoning == nil
+		persistDisableTagPrefill := c.ReasoningConfig.DisableReasoningTagPrefill == nil
 		config.DetectThinkingSupportFromBackend(ctx, c, inferenceModel, modelOpts)
 		// Update the config in the loader so it persists for future requests
-		cl.UpdateModelConfig(c.Name, func(cfg *config.ModelConfig) {
-			cfg.ReasoningConfig.DisableReasoning = c.ReasoningConfig.DisableReasoning
-			cfg.ReasoningConfig.DisableReasoningTagPrefill = c.ReasoningConfig.DisableReasoningTagPrefill
-			if c.MediaMarker != "" {
-				cfg.MediaMarker = c.MediaMarker
-			}
-		})
+		persistProbedReasoning(cl, c.Name, c, persistDisableReasoning, persistDisableTagPrefill)
 	}
 
 	var protoMessages []*proto.Message

core/backend/llm_probe_test.go

@@ -1,6 +1,8 @@
 package backend
 
 import (
+	"os"
+
 	"github.com/mudler/LocalAI/core/config"
 
 	"github.com/gpustack/gguf-parser-go/util/ptr"
@@ -27,3 +29,90 @@ var _ = Describe("thinking probe gating", func() {
 		Expect(needsThinkingProbe(cfg)).To(BeFalse())
 	})
 })
+
+var _ = Describe("persistProbedReasoning", func() {
+	const modelName = "probe-test"
+
+	// newLoaderWithConfig seeds a ModelConfigLoader with a single model config
+	// parsed from yamlBody, mirroring how the loader is populated from disk.
+	newLoaderWithConfig := func(yamlBody string) *config.ModelConfigLoader {
+		tmp, err := os.CreateTemp("", "persist-probed-reasoning-*.yaml")
+		Expect(err).ToNot(HaveOccurred())
+		defer func() { _ = os.Remove(tmp.Name()) }()
+
+		_, err = tmp.WriteString(yamlBody)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(tmp.Close()).To(Succeed())
+
+		cl := config.NewModelConfigLoader("")
+		Expect(cl.ReadModelConfig(tmp.Name())).To(Succeed())
+		return cl
+	}
+
+	It("persists a reasoning slot the probe was allowed to fill (was nil beforehand)", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false) // backend detected: supports thinking
+		probed.ReasoningConfig.DisableReasoningTagPrefill = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, true, true)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeFalse())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeTrue())
+	})
+
+	It("does not persist a slot that already carried a request-scoped value before the probe ran", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Simulates ApplyReasoningEffort("none") having set this on the
+		// request-scoped copy before the probe ran - not a genuine backend
+		// detection, so it must never reach the persisted config (#10622).
+		probed.ReasoningConfig.DisableReasoning = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).To(BeNil())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeNil())
+	})
+
+	It("preserves an operator's explicit persisted disable when the guard is false", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\nreasoning:\n  disable: true\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Even if the request-scoped copy ends up holding a different value,
+		// persistDisableReasoning=false must keep the operator's own setting.
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeTrue())
+	})
+
+	It("persists the media marker regardless of the reasoning guards", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.MediaMarker = "<__media__>"
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.MediaMarker).To(Equal("<__media__>"))
+	})
+})

core/config/context_fit.go

@@ -0,0 +1,144 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/xsysinfo"
+	"github.com/mudler/xlog"
+)
+
+// contextFitHeadroomDivisor reserves a slice of per-device VRAM as headroom when
+// deciding whether an auto-derived context fits. The gguf-parser footprint
+// already covers weights + KV + compute buffer, but a live load also pays for
+// allocator fragmentation, the CUDA/HIP context, and whatever else shares the
+// card, so we require the estimate to leave at least 1/divisor of the device
+// free. /5 (~20% headroom) mirrors the SWA full-cache gate's margin.
+const contextFitHeadroomDivisor = 5
+
+// contextFitCandidates is the descending set of context windows tried when the
+// DefaultAutoContextSize cap itself does not fit per-device VRAM. Only the rare
+// big-model-on-tiny-card case reaches this walk; it is capped at the base
+// choice and floored at DefaultContextSize, and returns the first (largest)
+// candidate that fits.
+var contextFitCandidates = []int{8192, 6144, 4096}
+
+// perDeviceVRAM reports the smallest per-GPU VRAM ceiling in bytes (0 = unknown
+// or no GPU). It is a package var so tests can inject a deterministic value —
+// detection does a live GPU probe. Per-device (not summed) is the right budget:
+// with all layers offloaded to a single device the whole footprint must fit that
+// one card, and a multi-GPU host is bounded by its smallest card. This mirrors
+// localGPU's use of MinPerGPUVRAM in hardware_defaults.go.
+var perDeviceVRAM = func() uint64 {
+	v, _ := xsysinfo.MinPerGPUVRAM()
+	return v
+}
+
+// estimateContextVRAM returns the estimated per-device VRAM footprint (bytes) of
+// running f fully offloaded at ctx tokens — weights + KV cache + compute buffer.
+// It returns 0 when it cannot produce an estimate (nil file, no tensors, or a
+// parser panic), which the caller treats as "cannot confirm a smaller fit" and
+// so keeps the conservative cap rather than clamping on a bogus number. It is a
+// package var so tests can stub it (a fabricated GGUF carries no tensors and
+// estimates to ~0).
+var estimateContextVRAM = func(f *gguf.GGUFFile, ctx int) (footprint uint64) {
+	if f == nil {
+		return 0
+	}
+	if ctx <= 0 {
+		ctx = DefaultContextSize
+	}
+	// The gguf-parser estimator panics on degenerate / partially-parsed GGUFs;
+	// treat any failure as "unknown" so config loading never crashes on a model
+	// the parser mis-handles.
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Debug("[context_fit] per-device VRAM estimate failed; treating as unknown", "error", r)
+			footprint = 0
+		}
+	}()
+	// Offload all layers (LocalAI's DefaultNGPULayers default; the estimator
+	// clamps to the model's block count) so the estimate reflects a fully
+	// GPU-resident model. NonUMA is the discrete-GPU figure (larger than the UMA
+	// one), which keeps the fit check conservative on unified-memory hosts — they
+	// have ample memory to clear it anyway.
+	est := f.EstimateLLaMACppRun(
+		gguf.WithLLaMACppContextSize(int32(ctx)),
+		gguf.WithLLaMACppOffloadLayers(uint64(DefaultNGPULayers)),
+	)
+	sum := est.Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return 0
+	}
+	var total uint64
+	for _, v := range sum.Items[0].VRAMs {
+		total += uint64(v.NonUMA)
+	}
+	return total
+}
+
+// contextFitsVRAM reports whether an estimated footprint fits a per-device VRAM
+// ceiling with headroom (VRAM must exceed the footprint by ~1/divisor). Unknown
+// inputs (0) are treated as "cannot confirm" so a detection or estimate gap does
+// not clamp the context.
+func contextFitsVRAM(footprint, vram uint64) bool {
+	if footprint == 0 || vram == 0 {
+		return false
+	}
+	return vram >= footprint+footprint/contextFitHeadroomDivisor
+}
+
+// autoContextSize picks the default context to use for f when the user did not
+// set context_size. The choice is deliberately conservative, NOT
+// VRAM-maximizing:
+//
+//  1. Base cap: min(trainedMax, DefaultAutoContextSize). A small model keeps its
+//     trained window; a long-context model (128k / 256k / 1M) is capped so its
+//     KV cache does not default to a size no consumer GPU can hold. This applies
+//     always, including CPU / unknown-VRAM hosts.
+//  2. VRAM is only a downward safety: when a per-device VRAM ceiling IS detected
+//     and even the base cap would not fit it (with headroom), step down through
+//     contextFitCandidates to the largest window that fits, floored at
+//     DefaultContextSize. When VRAM is unknown we skip this — the base cap is
+//     already safe and we must not regress CPU / detection-gap hosts.
+//
+// trainedMax <= 0 means the estimate yielded nothing usable; the caller keeps
+// its existing DefaultContextSize fallback in that case, so this is only called
+// with a positive trainedMax.
+func autoContextSize(f *gguf.GGUFFile, trainedMax int) int {
+	chosen := trainedMax
+	if chosen > DefaultAutoContextSize {
+		chosen = DefaultAutoContextSize
+	}
+
+	vram := perDeviceVRAM()
+	if vram == 0 {
+		// No per-device VRAM detected (CPU-only, unified memory reporting nothing,
+		// or a detection gap). The bug is GPU OOM-on-load, so with no GPU budget to
+		// reason about we must not clamp — the base cap already bounds long-context
+		// models.
+		return chosen
+	}
+
+	if contextFitsVRAM(estimateContextVRAM(f, chosen), vram) {
+		return chosen
+	}
+
+	// The base cap does not fit this card. Walk candidates downward and take the
+	// largest that fits, never below DefaultContextSize.
+	for _, cand := range contextFitCandidates {
+		if cand > chosen || cand < DefaultContextSize {
+			continue
+		}
+		if contextFitsVRAM(estimateContextVRAM(f, cand), vram) {
+			xlog.Debug("[context_fit] capped auto context to fit per-device VRAM",
+				"context", cand, "base_cap", chosen, "vram_gib", vram>>30)
+			return cand
+		}
+	}
+
+	// Nothing fit (an unusually large model on a tiny card): fall back to the
+	// floor. The backend still clamps n_gpu_layers to what fits, so a partial
+	// offload can keep the model loadable rather than aborting outright.
+	xlog.Debug("[context_fit] no candidate context fit per-device VRAM; using floor",
+		"context", DefaultContextSize, "base_cap", chosen, "vram_gib", vram>>30)
+	return DefaultContextSize
+}

core/config/context_fit_internal_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These specs exercise the auto-derived default context. The detection seams
+// (perDeviceVRAM, estimateContextVRAM) are package vars so a deterministic VRAM
+// ceiling and footprint can be injected without a real GPU or model file — the
+// same pattern hardware_defaults_internal_test.go uses for localGPU.
+var _ = Describe("Auto-derived default context (VRAM-aware cap)", func() {
+	const gib = uint64(1) << 30
+
+	var (
+		origVRAM     func() uint64
+		origEstimate func(f *gguf.GGUFFile, ctx int) uint64
+	)
+
+	BeforeEach(func() {
+		origVRAM = perDeviceVRAM
+		origEstimate = estimateContextVRAM
+	})
+	AfterEach(func() {
+		perDeviceVRAM = origVRAM
+		estimateContextVRAM = origEstimate
+	})
+
+	Context("autoContextSize", func() {
+		It("caps a long-context model at DefaultAutoContextSize when VRAM is ample", func() {
+			// 1M-context model on an 80 GiB card: we do NOT chase the trained max,
+			// we keep the conservative 8k cap (users opt into more via context_size).
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib } // trivially fits
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+
+		It("keeps a small model's trained window instead of inflating it", func() {
+			// trained 4096 < 8192: min() keeps 4096, it is not raised to the cap.
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib }
+			Expect(autoContextSize(nil, 4096)).To(Equal(4096))
+		})
+
+		It("steps below the cap when even 8k would not fit a tiny card", func() {
+			// A large model on a 2 GiB card where the 8k footprint overflows but a
+			// smaller context fits: choose the largest that fits, never below the
+			// floor. Footprint grows with context so the walk finds a fit.
+			perDeviceVRAM = func() uint64 { return 2 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, ctx int) uint64 {
+				return gib + uint64(ctx)*100000
+			}
+			chosen := autoContextSize(nil, 1048576)
+			Expect(chosen).To(BeNumerically("<", DefaultAutoContextSize))
+			Expect(chosen).To(BeNumerically(">=", DefaultContextSize))
+			// The chosen context's footprint must actually fit the card with headroom.
+			Expect(contextFitsVRAM(estimateContextVRAM(nil, chosen), 2*gib)).To(BeTrue())
+		})
+
+		It("falls back to the floor when nothing fits", func() {
+			// Even DefaultContextSize does not fit: return the floor and let the
+			// backend clamp n_gpu_layers to what it can (partial offload) rather
+			// than defaulting to a window guaranteed to abort.
+			perDeviceVRAM = func() uint64 { return 1 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 100 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultContextSize))
+		})
+
+		It("does not clamp when per-device VRAM is unknown", func() {
+			// CPU-only / detection gap: no GPU budget to reason about, so we must
+			// not regress — keep the conservative base cap regardless of estimate.
+			perDeviceVRAM = func() uint64 { return 0 }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 999 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+	})
+
+	Context("guessGGUFFromFile", func() {
+		It("never overrides an explicitly configured context_size", func() {
+			// A fabricated GGUF is enough: the context branch is skipped entirely
+			// when the user pinned context_size, so the estimate is never consulted.
+			explicit := 262144
+			cfg := &ModelConfig{LLMConfig: LLMConfig{ContextSize: &explicit}}
+			f := &gguf.GGUFFile{
+				Header: gguf.GGUFHeader{
+					MetadataKV: gguf.GGUFMetadataKVs{
+						{
+							Key:       "general.architecture",
+							ValueType: gguf.GGUFMetadataValueTypeString,
+							Value:     "llama",
+						},
+					},
+				},
+			}
+			guessGGUFFromFile(cfg, f, 0)
+			Expect(cfg.ContextSize).ToNot(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(262144))
+		})
+	})
+})

core/config/defaults.go

@@ -18,6 +18,18 @@ const (
 	// safe default beats a tiny, surprising window that truncates real prompts.
 	DefaultContextSize = 4096
 
+	// DefaultAutoContextSize caps the context we auto-derive from a GGUF when the
+	// user did not set context_size. The GGUF importer used to default a model's
+	// context to its full trained window (n_ctx_train). For long-context models
+	// (128k / 256k / 1M) that KV cache cannot fit a consumer GPU and the backend
+	// aborts on load (exitCode=-1) even though the model file is fine. So instead
+	// of shooting for the trained max, we keep a modest default: a small model
+	// (trained < this) keeps its trained window, while a long-context model caps
+	// here. Users who want the full window raise context_size explicitly. This is
+	// a conservative default, not a VRAM-maximizing one — VRAM is only used to
+	// step further DOWN when even this cap would not fit (see context_fit.go).
+	DefaultAutoContextSize = 8192
+
 	// DefaultNGPULayers means "offload all layers"; the backend (fit_params)
 	// clamps to what actually fits in device memory.
 	DefaultNGPULayers = 99999999

core/config/gguf.go

@@ -28,9 +28,14 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 
 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
 	if defaultCtx == 0 && cfg.ContextSize == nil {
-		ctxSize := f.EstimateLLaMACppRun().ContextSize
-		if ctxSize > 0 {
-			cSize := int(ctxSize)
+		// trainedMax is the model's full trained context window (n_ctx_train).
+		// Defaulting a model to it unbounded is what OOMs long-context models at
+		// load: a 128k / 256k / 1M KV cache cannot fit a consumer GPU and the
+		// backend aborts (exitCode=-1). autoContextSize instead caps to a modest
+		// default and only steps below it when detected per-device VRAM demands.
+		trainedMax := int(f.EstimateLLaMACppRun().ContextSize)
+		if trainedMax > 0 {
+			cSize := autoContextSize(f, trainedMax)
 			cfg.ContextSize = &cSize
 		} else {
 			defaultCtx = DefaultContextSize

core/config/model_capabilities.go

@@ -0,0 +1,197 @@
+package config
+
+// This file is the single source of truth for deriving a model's user-facing
+// capabilities and input/output modalities from its ModelConfig. Both the
+// OpenAI-compatible /v1/models/capabilities endpoint and the Ollama-compatible
+// /api/tags|/api/show surface consume these, so the vocabulary stays consistent
+// across clients. Keep the detection heuristics here rather than duplicating
+// them per endpoint.
+
+// VisionSupported reports whether the model can accept image inputs.
+//
+// We deliberately avoid HasUsecases(FLAG_VISION): GuessUsecases has no
+// FLAG_VISION branch and reports true for any chat model, so it would paint
+// vision onto text-only models. Instead we look for explicit signals: the
+// declared KnownUsecases bit, a multimodal projector, or a template/backend
+// multimodal marker.
+func (c *ModelConfig) VisionSupported() bool {
+	if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION {
+		return true
+	}
+	if c.MMProj != "" {
+		return true
+	}
+	if c.TemplateConfig.Multimodal != "" {
+		return true
+	}
+	if c.MediaMarker != "" {
+		return true
+	}
+	return false
+}
+
+// ToolSupported reports whether the model is wired up for tool / function
+// calling. We look for any of the explicit knobs LocalAI uses to drive
+// function-call extraction (regex match, response regex, grammar triggers, XML
+// format) or the auto-detected tool-format markers the llama.cpp backend
+// populates during model load.
+func (c *ModelConfig) ToolSupported() bool {
+	fc := c.FunctionsConfig
+	if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
+		return true
+	}
+	if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
+		return true
+	}
+	if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
+		return true
+	}
+	if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
+		return true
+	}
+	return false
+}
+
+// ThinkingSupported reports whether the model has reasoning / thinking enabled.
+// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
+// when the backend probe reports that the model supports thinking.
+func (c *ModelConfig) ThinkingSupported() bool {
+	rc := c.ReasoningConfig
+	if rc.DisableReasoning != nil && !*rc.DisableReasoning {
+		return true
+	}
+	if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
+		// Explicit thinking markers imply support unless explicitly disabled.
+		return rc.DisableReasoning == nil || !*rc.DisableReasoning
+	}
+	return false
+}
+
+// AudioInputSupported reports whether a chat/generation model accepts audio as
+// input (e.g. vLLM omni models). The signal is the vLLM per-prompt audio limit;
+// there is no FLAG_* for "chat model that hears audio", which is exactly why a
+// plain usecase list can't express it. Transcription models are handled
+// separately in InputModalities via FLAG_TRANSCRIPT.
+func (c *ModelConfig) AudioInputSupported() bool {
+	return c.LimitMMPerPrompt.LimitAudioPerPrompt > 0
+}
+
+// VideoInputSupported reports whether a chat/generation model accepts video as
+// input. The signal is the vLLM per-prompt video limit. Note this is distinct
+// from FLAG_VIDEO, which denotes video *generation* (diffusers) — an output
+// modality, not an input one.
+func (c *ModelConfig) VideoInputSupported() bool {
+	return c.LimitMMPerPrompt.LimitVideoPerPrompt > 0
+}
+
+// Capabilities returns the ordered list of capability strings the model
+// supports, using the canonical usecase vocabulary (chat, vision, transcript,
+// tts, embeddings, image, video, ...) plus the modifier capabilities "tools"
+// and "thinking". Vision is resolved via VisionSupported (not HasUsecases) to
+// avoid the guess-heuristic false positive.
+func (c *ModelConfig) Capabilities() []string {
+	chat := c.HasUsecases(FLAG_CHAT)
+	completion := c.HasUsecases(FLAG_COMPLETION)
+
+	var caps []string
+	add := func(cond bool, name string) {
+		if cond {
+			caps = append(caps, name)
+		}
+	}
+
+	add(chat, UsecaseChat)
+	add(completion, UsecaseCompletion)
+	add(c.HasUsecases(FLAG_EDIT), UsecaseEdit)
+	add(c.HasUsecases(FLAG_EMBEDDINGS), UsecaseEmbeddings)
+	add(c.HasUsecases(FLAG_RERANK), UsecaseRerank)
+	// Vision is only meaningful as an image-understanding modifier on a chat/
+	// completion model. Gating on (chat||completion) matches the Ollama surface
+	// and avoids a false positive when config defaults hydrate a MediaMarker on
+	// a non-chat model (e.g. a pure ASR/TTS backend).
+	add((chat || completion) && c.VisionSupported(), UsecaseVision)
+	// tools/thinking are modifiers on the chat/completion surface.
+	add((chat || completion) && c.ToolSupported(), "tools")
+	add((chat || completion) && c.ThinkingSupported(), "thinking")
+	add(c.HasUsecases(FLAG_TRANSCRIPT), UsecaseTranscript)
+	add(c.HasUsecases(FLAG_TTS), UsecaseTTS)
+	add(c.HasUsecases(FLAG_SOUND_GENERATION), UsecaseSoundGeneration)
+	add(c.HasUsecases(FLAG_IMAGE), UsecaseImage)
+	add(c.HasUsecases(FLAG_VIDEO), UsecaseVideo)
+	add(c.HasUsecases(FLAG_VAD), UsecaseVAD)
+	add(c.HasUsecases(FLAG_DETECTION), UsecaseDetection)
+	add(c.HasUsecases(FLAG_DEPTH), UsecaseDepth)
+	add(c.HasUsecases(FLAG_AUDIO_TRANSFORM), UsecaseAudioTransform)
+	add(c.HasUsecases(FLAG_DIARIZATION), UsecaseDiarization)
+	add(c.HasUsecases(FLAG_SOUND_CLASSIFICATION), UsecaseSoundClassification)
+	add(c.HasUsecases(FLAG_REALTIME_AUDIO), UsecaseRealtimeAudio)
+	add(c.HasUsecases(FLAG_FACE_RECOGNITION), UsecaseFaceRecognition)
+	add(c.HasUsecases(FLAG_SPEAKER_RECOGNITION), UsecaseSpeakerRecognition)
+	return caps
+}
+
+// InputModalities returns the set of modalities (text, image, audio, video) the
+// model accepts as input, ordered text→image→audio→video. This is what an
+// attachment router consults to decide whether an image/audio/video file can be
+// handed to the active model directly.
+func (c *ModelConfig) InputModalities() []string {
+	imageGen := c.HasUsecases(FLAG_IMAGE)
+	videoGen := c.HasUsecases(FLAG_VIDEO)
+	chatish := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION)
+
+	textIn := chatish || c.HasUsecases(FLAG_EDIT) ||
+		c.HasUsecases(FLAG_EMBEDDINGS) || c.HasUsecases(FLAG_RERANK) || c.HasUsecases(FLAG_TOKENIZE) ||
+		c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || imageGen || videoGen
+
+	// Image input via a chat model requires vision (gated on chat, like the
+	// Ollama surface); detection/depth/face models consume images directly.
+	imageIn := (chatish && c.VisionSupported()) || c.LimitMMPerPrompt.LimitImagePerPrompt > 0 ||
+		c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION)
+
+	audioIn := c.AudioInputSupported() || c.HasUsecases(FLAG_TRANSCRIPT) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) ||
+		c.HasUsecases(FLAG_REALTIME_AUDIO) || c.HasUsecases(FLAG_VAD) || c.HasUsecases(FLAG_DIARIZATION) ||
+		c.HasUsecases(FLAG_SOUND_CLASSIFICATION) || c.HasUsecases(FLAG_SPEAKER_RECOGNITION)
+
+	videoIn := c.VideoInputSupported()
+
+	var mods []string
+	if textIn {
+		mods = append(mods, "text")
+	}
+	if imageIn {
+		mods = append(mods, "image")
+	}
+	if audioIn {
+		mods = append(mods, "audio")
+	}
+	if videoIn {
+		mods = append(mods, "video")
+	}
+	return mods
+}
+
+// OutputModalities returns the set of modalities (text, image, audio, video)
+// the model produces, ordered text→image→audio→video.
+func (c *ModelConfig) OutputModalities() []string {
+	textOut := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) || c.HasUsecases(FLAG_EDIT) ||
+		c.HasUsecases(FLAG_TRANSCRIPT)
+	imageOut := c.HasUsecases(FLAG_IMAGE)
+	audioOut := c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) ||
+		c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO)
+	videoOut := c.HasUsecases(FLAG_VIDEO)
+
+	var mods []string
+	if textOut {
+		mods = append(mods, "text")
+	}
+	if imageOut {
+		mods = append(mods, "image")
+	}
+	if audioOut {
+		mods = append(mods, "audio")
+	}
+	if videoOut {
+		mods = append(mods, "video")
+	}
+	return mods
+}

core/config/model_capabilities_test.go

@@ -0,0 +1,103 @@
+package config
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func usecaseBits(flags ModelConfigUsecase) *ModelConfigUsecase {
+	return &flags
+}
+
+var _ = Describe("Model capabilities derivation", func() {
+	Describe("VisionSupported", func() {
+		It("is false for a plain text chat model", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+
+		It("is true when the FLAG_VISION bit is declared", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT | FLAG_VISION), Backend: "llama.cpp"}
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
+		It("is true when an mmproj projector is set", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			cfg.MMProj = "mmproj.gguf" // promoted field from the embedded options struct
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
+		It("does not fall for the GuessUsecases FLAG_VISION false positive", func() {
+			// A chat model with a chat template would make HasUsecases(FLAG_VISION)
+			// return true via the guess heuristic; VisionSupported must not.
+			cfg := &ModelConfig{Backend: "llama.cpp"}
+			cfg.TemplateConfig.Chat = "{{.Input}}"
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+	})
+
+	Describe("AudioInputSupported / VideoInputSupported", func() {
+		It("detects vLLM omni audio input via limit_mm_per_prompt", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1
+			Expect(cfg.AudioInputSupported()).To(BeTrue())
+			Expect(cfg.VideoInputSupported()).To(BeFalse())
+		})
+
+		It("detects vLLM omni video input via limit_mm_per_prompt", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitVideoPerPrompt = 2
+			Expect(cfg.VideoInputSupported()).To(BeTrue())
+		})
+	})
+
+	Describe("Capabilities + modalities", func() {
+		It("a text-only chat model exposes chat and text-only modalities", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseChat))
+			Expect(cfg.Capabilities()).NotTo(ContainElement(UsecaseVision))
+			Expect(cfg.Capabilities()).NotTo(ContainElement(UsecaseTranscript))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("a vision chat model accepts text+image input", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT | FLAG_VISION), Backend: "llama.cpp"}
+			Expect(cfg.Capabilities()).To(ContainElements(UsecaseChat, UsecaseVision))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text", "image"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("an omni chat model accepts text+audio input without an audio capability flag", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
+			cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1
+			// audio-in is a modality, not a usecase string — this is exactly the
+			// case a plain capability list cannot express.
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseChat))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text", "audio"}))
+		})
+
+		It("a transcription model reads audio and writes text", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TRANSCRIPT), Backend: "parakeet-cpp"}
+			Expect(cfg.Capabilities()).To(Equal([]string{UsecaseTranscript}))
+			Expect(cfg.InputModalities()).To(Equal([]string{"audio"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"text"}))
+		})
+
+		It("an image-generation model reads text and writes an image", func() {
+			// stablediffusion-ggml is image-only; plain "stablediffusion" is also
+			// in GuessUsecases' video-backend list, so it would report video too.
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_IMAGE), Backend: "stablediffusion-ggml"}
+			Expect(cfg.Capabilities()).To(Equal([]string{UsecaseImage}))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"image"}))
+		})
+
+		It("a TTS model reads text and writes audio", func() {
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "piper"}
+			Expect(cfg.Capabilities()).To(ContainElement(UsecaseTTS))
+			Expect(cfg.InputModalities()).To(Equal([]string{"text"}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{"audio"}))
+		})
+	})
+})

core/http/auth/oauth.go

@@ -202,6 +202,11 @@ func (m *OAuthManager) CallbackHandler(providerName string, db *gorm.DB, adminEm
 			userInfo, err = fetchGitHubUserInfoAsOAuth(ctx, token.AccessToken)
 		}
 		if err != nil {
+			// Surface the real cause server-side: ID-token verify failures (issuer/
+			// audience mismatch behind a reverse proxy), a missing id_token, claim
+			// parse errors, or the GitHub userinfo HTTP status/body. The client still
+			// gets the generic message below; details go to logs only. See #10677.
+			xlog.Error("OAuth callback: failed to resolve user info", "provider", providerName, "error", err)
 			return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to fetch user info"})
 		}
 

core/http/endpoints/ollama/capabilities.go

@@ -49,62 +49,23 @@ func modelCapabilities(cfg *config.ModelConfig) []string {
 	return caps
 }
 
-// hasVisionSupport reports whether the model can accept image inputs. We avoid
-// cfg.HasUsecases(FLAG_VISION) because GuessUsecases has no FLAG_VISION case
-// and returns true for any chat model — see core/config/model_config.go. Instead
-// we look for explicit signals: KnownUsecases bit, multimodal projector, or
-// template/backend-reported multimodal markers.
+// hasVisionSupport reports whether the model can accept image inputs.
+// The detection heuristic is the canonical config.ModelConfig.VisionSupported —
+// kept as a thin wrapper here so the Ollama capability mapping reads cleanly.
 func hasVisionSupport(cfg *config.ModelConfig) bool {
-	if cfg.KnownUsecases != nil && (*cfg.KnownUsecases&config.FLAG_VISION) == config.FLAG_VISION {
-		return true
-	}
-	if cfg.MMProj != "" {
-		return true
-	}
-	if cfg.TemplateConfig.Multimodal != "" {
-		return true
-	}
-	if cfg.MediaMarker != "" {
-		return true
-	}
-	return false
+	return cfg.VisionSupported()
 }
 
-// hasToolSupport reports whether the model is wired up for tool / function calling.
-// We look for any of the explicit configuration knobs LocalAI uses to drive
-// function-call extraction (regex match, response regex, grammar triggers, XML
-// format) or for the auto-detected tool-format markers populated by the
-// llama.cpp backend during model load.
+// hasToolSupport reports whether the model is wired up for tool / function
+// calling. Delegates to the canonical config.ModelConfig.ToolSupported.
 func hasToolSupport(cfg *config.ModelConfig) bool {
-	fc := cfg.FunctionsConfig
-	if fc.ToolFormatMarkers != nil && fc.ToolFormatMarkers.FormatType != "" {
-		return true
-	}
-	if len(fc.JSONRegexMatch) > 0 || len(fc.ResponseRegex) > 0 {
-		return true
-	}
-	if fc.XMLFormatPreset != "" || fc.XMLFormat != nil {
-		return true
-	}
-	if len(fc.GrammarConfig.GrammarTriggers) > 0 || fc.GrammarConfig.SchemaType != "" {
-		return true
-	}
-	return false
+	return cfg.ToolSupported()
 }
 
 // hasThinkingSupport reports whether the model has reasoning / thinking enabled.
-// LocalAI sets DisableReasoning=false (or leaves thinking markers configured)
-// when the backend probe reports that the model supports thinking.
+// Delegates to the canonical config.ModelConfig.ThinkingSupported.
 func hasThinkingSupport(cfg *config.ModelConfig) bool {
-	rc := cfg.ReasoningConfig
-	if rc.DisableReasoning != nil && !*rc.DisableReasoning {
-		return true
-	}
-	if len(rc.ThinkingStartTokens) > 0 || len(rc.TagPairs) > 0 {
-		// Explicit thinking markers imply support unless explicitly disabled.
-		return rc.DisableReasoning == nil || !*rc.DisableReasoning
-	}
-	return false
+	return cfg.ThinkingSupported()
 }
 
 // quantRegex matches GGUF-style quantization suffixes (Q4_K_M, Q8_0, IQ3_XS, F16, ...).

core/http/endpoints/openai/list.go

@@ -21,48 +21,11 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, ap
 		authDB = db[0]
 	}
 	return func(c echo.Context) error {
-		// If blank, no filter is applied.
-		filter := c.QueryParam("filter")
-
-		// By default, exclude any loose files that are already referenced by a configuration file.
-		var policy galleryop.LooseFilePolicy
-		excludeConfigured := c.QueryParam("excludeConfigured")
-		if excludeConfigured == "" || excludeConfigured == "true" {
-			policy = galleryop.SKIP_IF_CONFIGURED
-		} else {
-			policy = galleryop.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user?
-		}
-
-		filterFn, err := config.BuildNameFilterFn(filter)
+		modelNames, err := listVisibleModelNames(c, bcl, ml, authDB)
 		if err != nil {
 			return err
 		}
 
-		modelNames, err := galleryop.ListModels(bcl, ml, filterFn, policy)
-		if err != nil {
-			return err
-		}
-
-		// Filter models by user's allowlist if auth is enabled
-		if authDB != nil {
-			if user := auth.GetUser(c); user != nil && user.Role != auth.RoleAdmin {
-				perm, err := auth.GetCachedUserPermissions(c, authDB, user.ID)
-				if err == nil && perm.AllowedModels.Enabled {
-					allowed := map[string]bool{}
-					for _, m := range perm.AllowedModels.Models {
-						allowed[m] = true
-					}
-					filtered := make([]string, 0, len(modelNames))
-					for _, m := range modelNames {
-						if allowed[m] {
-							filtered = append(filtered, m)
-						}
-					}
-					modelNames = filtered
-				}
-			}
-		}
-
 		// Map from a slice of names to a slice of OpenAIModel response objects
 		dataModels := []schema.OpenAIModel{}
 		for _, m := range modelNames {
@@ -75,3 +38,53 @@ func ListModelsEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, ap
 		})
 	}
 }
+
+// listVisibleModelNames resolves the model names visible to the caller, applying
+// the same query filters (filter, excludeConfigured) and per-user allowlist as
+// the OpenAI models listing. Shared by ListModelsEndpoint and
+// ListModelCapabilitiesEndpoint so both stay consistent.
+func listVisibleModelNames(c echo.Context, bcl *config.ModelConfigLoader, ml *model.ModelLoader, authDB *gorm.DB) ([]string, error) {
+	// If blank, no filter is applied.
+	filter := c.QueryParam("filter")
+
+	// By default, exclude any loose files that are already referenced by a configuration file.
+	var policy galleryop.LooseFilePolicy
+	excludeConfigured := c.QueryParam("excludeConfigured")
+	if excludeConfigured == "" || excludeConfigured == "true" {
+		policy = galleryop.SKIP_IF_CONFIGURED
+	} else {
+		policy = galleryop.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user?
+	}
+
+	filterFn, err := config.BuildNameFilterFn(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	modelNames, err := galleryop.ListModels(bcl, ml, filterFn, policy)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter models by user's allowlist if auth is enabled
+	if authDB != nil {
+		if user := auth.GetUser(c); user != nil && user.Role != auth.RoleAdmin {
+			perm, err := auth.GetCachedUserPermissions(c, authDB, user.ID)
+			if err == nil && perm.AllowedModels.Enabled {
+				allowed := map[string]bool{}
+				for _, m := range perm.AllowedModels.Models {
+					allowed[m] = true
+				}
+				filtered := make([]string, 0, len(modelNames))
+				for _, m := range modelNames {
+					if allowed[m] {
+						filtered = append(filtered, m)
+					}
+				}
+				modelNames = filtered
+			}
+		}
+	}
+
+	return modelNames, nil
+}

core/http/endpoints/openai/list_capabilities.go

@@ -0,0 +1,50 @@
+package openai
+
+import (
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	model "github.com/mudler/LocalAI/pkg/model"
+	"gorm.io/gorm"
+)
+
+// ListModelCapabilitiesEndpoint is a LocalAI-specific extension of the OpenAI
+// models listing. It returns the same set of models as /v1/models but enriches
+// each entry with the capabilities and input/output modalities the model
+// supports, so clients can decide whether an image/audio/video attachment can be
+// handed to a given model directly (or must be converted/transcribed first).
+//
+// It is purely additive: clients that don't know about it keep using /v1/models
+// and see no change.
+// @Summary List available models enriched with capabilities and input/output modalities.
+// @Tags models
+// @Success 200 {object} schema.ModelCapabilitiesResponse "Response"
+// @Router /v1/models/capabilities [get]
+func ListModelCapabilitiesEndpoint(bcl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, db ...*gorm.DB) echo.HandlerFunc {
+	var authDB *gorm.DB
+	if len(db) > 0 {
+		authDB = db[0]
+	}
+	return func(c echo.Context) error {
+		modelNames, err := listVisibleModelNames(c, bcl, ml, authDB)
+		if err != nil {
+			return err
+		}
+
+		dataModels := []schema.ModelCapabilities{}
+		for _, m := range modelNames {
+			entry := schema.ModelCapabilities{ID: m, Object: "model"}
+			if cfg, ok := bcl.GetModelConfig(m); ok {
+				entry.Capabilities = cfg.Capabilities()
+				entry.InputModalities = cfg.InputModalities()
+				entry.OutputModalities = cfg.OutputModalities()
+			}
+			dataModels = append(dataModels, entry)
+		}
+
+		return c.JSON(200, schema.ModelCapabilitiesResponse{
+			Object: "list",
+			Data:   dataModels,
+		})
+	}
+}

core/http/endpoints/openai/list_capabilities_test.go

@@ -0,0 +1,119 @@
+package openai
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ListModelCapabilitiesEndpoint", func() {
+	var (
+		e       *echo.Echo
+		tmpDir  string
+		bcl     *config.ModelConfigLoader
+		ml      *model.ModelLoader
+		appConf *config.ApplicationConfig
+	)
+
+	BeforeEach(func() {
+		var err error
+		e = echo.New()
+		tmpDir, err = os.MkdirTemp("", "models-caps-test-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		st, err := system.GetSystemState(system.WithModelPath(tmpDir))
+		Expect(err).NotTo(HaveOccurred())
+		ml = model.NewModelLoader(st)
+		bcl = config.NewModelConfigLoader(tmpDir)
+		appConf = config.NewApplicationConfig()
+	})
+
+	AfterEach(func() {
+		_ = os.RemoveAll(tmpDir)
+	})
+
+	writeConfig := func(name, yaml string) {
+		path := filepath.Join(tmpDir, name+".yaml")
+		Expect(os.WriteFile(path, []byte(yaml), 0o644)).To(Succeed())
+		Expect(bcl.ReadModelConfig(path)).To(Succeed())
+	}
+
+	// call exercises the endpoint with auth disabled (no auth DB), which is the
+	// standard deployment path. The per-user allowlist branch is shared verbatim
+	// with ListModelsEndpoint (listVisibleModelNames) and covered there.
+	call := func() schema.ModelCapabilitiesResponse {
+		req := httptest.NewRequest(http.MethodGet, "/v1/models/capabilities", nil)
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+
+		handler := ListModelCapabilitiesEndpoint(bcl, ml, appConf)
+		Expect(handler(c)).To(Succeed())
+		Expect(rec.Code).To(Equal(http.StatusOK))
+
+		var resp schema.ModelCapabilitiesResponse
+		Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed())
+		return resp
+	}
+
+	entryFor := func(resp schema.ModelCapabilitiesResponse, id string) *schema.ModelCapabilities {
+		for i := range resp.Data {
+			if resp.Data[i].ID == id {
+				return &resp.Data[i]
+			}
+		}
+		return nil
+	}
+
+	It("returns the list envelope even with no models", func() {
+		resp := call()
+		Expect(resp.Object).To(Equal("list"))
+	})
+
+	It("enriches a vision chat model with capabilities and image input modality", func() {
+		writeConfig("vlm", `
+name: vlm
+backend: llama-cpp
+known_usecases:
+  - FLAG_CHAT
+  - FLAG_VISION
+template:
+  chat: "{{ .Input }}"
+parameters:
+  model: qwen2.5-vl-Q4_K_M.gguf
+`)
+		entry := entryFor(call(), "vlm")
+		Expect(entry).NotTo(BeNil())
+		Expect(entry.Object).To(Equal("model"))
+		Expect(entry.Capabilities).To(ContainElements("chat", "vision"))
+		Expect(entry.InputModalities).To(ContainElements("text", "image"))
+		Expect(entry.OutputModalities).To(ContainElement("text"))
+	})
+
+	It("marks a parakeet model as an audio-in/text-out transcription model", func() {
+		writeConfig("parakeet", `
+name: parakeet
+backend: parakeet-cpp
+known_usecases:
+  - FLAG_TRANSCRIPT
+parameters:
+  model: parakeet-tdt-0.6b
+`)
+		entry := entryFor(call(), "parakeet")
+		Expect(entry).NotTo(BeNil())
+		Expect(entry.Capabilities).To(ContainElement("transcript"))
+		Expect(entry.InputModalities).To(Equal([]string{"audio"}))
+		Expect(entry.OutputModalities).To(Equal([]string{"text"}))
+		Expect(entry.Capabilities).NotTo(ContainElement("chat"))
+	})
+})

core/http/react-ui/src/App.css

@@ -6304,6 +6304,9 @@ select.input {
 .home-wizard {
   max-width: 48rem;
   width: 100%;
+  /* .home-page is a stretch column flex; a max-width child would otherwise
+     pin to the left cross-start edge. Center it. */
+  margin: 0 auto;
 }
 .home-wizard-hero {
   text-align: center;

core/http/routes/localai.go

@@ -272,25 +272,27 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 			"version": internal.PrintableVersion(),
 			// Flat endpoint list for backwards compatibility
 			"endpoints": map[string]any{
-				"models":           "/v1/models",
-				"chat_completions": "/v1/chat/completions",
-				"completions":      "/v1/completions",
-				"embeddings":       "/v1/embeddings",
-				"config_metadata":  "/api/models/config-metadata",
-				"config_json":      "/api/models/config-json/:name",
-				"config_patch":     "/api/models/config-json/:name",
-				"autocomplete":     "/api/models/config-metadata/autocomplete/:provider",
-				"vram_estimate":    "/api/models/vram-estimate",
-				"tts":              "/tts",
-				"transcription":    "/v1/audio/transcriptions",
-				"image_generation": "/v1/images/generations",
-				"swagger":          "/swagger/index.html",
-				"instructions":     "/api/instructions",
+				"models":              "/v1/models",
+				"models_capabilities": "/v1/models/capabilities",
+				"chat_completions":    "/v1/chat/completions",
+				"completions":         "/v1/completions",
+				"embeddings":          "/v1/embeddings",
+				"config_metadata":     "/api/models/config-metadata",
+				"config_json":         "/api/models/config-json/:name",
+				"config_patch":        "/api/models/config-json/:name",
+				"autocomplete":        "/api/models/config-metadata/autocomplete/:provider",
+				"vram_estimate":       "/api/models/vram-estimate",
+				"tts":                 "/tts",
+				"transcription":       "/v1/audio/transcriptions",
+				"image_generation":    "/v1/images/generations",
+				"swagger":             "/swagger/index.html",
+				"instructions":        "/api/instructions",
 			},
 			// Categorized endpoint groups for structured discovery
 			"endpoint_groups": map[string]any{
 				"openai_compatible": map[string]string{
 					"models":               "/v1/models",
+					"models_capabilities":  "/v1/models/capabilities",
 					"chat_completions":     "/v1/chat/completions",
 					"completions":          "/v1/completions",
 					"embeddings":           "/v1/embeddings",

core/http/routes/openai.go

@@ -257,4 +257,10 @@ func RegisterOpenAIRoutes(app *echo.Echo,
 	// List models
 	app.GET("/v1/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB()))
 	app.GET("/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB()))
+
+	// List models enriched with capabilities + input/output modalities
+	// (LocalAI-specific, additive superset of /v1/models).
+	capabilitiesHandler := openai.ListModelCapabilitiesEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.AuthDB())
+	app.GET("/v1/models/capabilities", capabilitiesHandler)
+	app.GET("/models/capabilities", capabilitiesHandler)
 }

core/schema/openai.go

@@ -251,3 +251,27 @@ type ModelsDataResponse struct {
 	Object string        `json:"object"`
 	Data   []OpenAIModel `json:"data"`
 }
+
+// ModelCapabilities is a strict superset of OpenAIModel that additionally
+// describes what a model can do and which modalities it accepts/produces. It is
+// served by the LocalAI-specific /v1/models/capabilities endpoint so clients can
+// route attachments (image/audio/video) to a model only when it can handle them.
+type ModelCapabilities struct {
+	ID     string `json:"id"`
+	Object string `json:"object"`
+	// Capabilities are canonical usecase strings (e.g. chat, vision, transcript,
+	// tts, embeddings, image, video) plus the modifiers "tools" and "thinking".
+	Capabilities []string `json:"capabilities"`
+	// InputModalities is the subset of {text,image,audio,video} the model accepts.
+	InputModalities []string `json:"input_modalities"`
+	// OutputModalities is the subset of {text,image,audio,video} the model produces.
+	OutputModalities []string `json:"output_modalities"`
+}
+
+// ModelCapabilitiesResponse is the envelope returned by /v1/models/capabilities.
+// It mirrors ModelsDataResponse so a client can treat it as an enriched
+// drop-in for /v1/models.
+type ModelCapabilitiesResponse struct {
+	Object string              `json:"object"`
+	Data   []ModelCapabilities `json:"data"`
+}

core/services/agentpool/agent_pool.go

@@ -426,7 +426,15 @@ func (s *AgentPoolService) Chat(name, message string) (string, error) {
 
 	// Process asynchronously
 	go func() {
+		started := time.Now()
 		response := ag.Ask(coreTypes.WithText(message))
+		outcome := "completed"
+		if response == nil {
+			outcome = "cancelled"
+		} else if response.Error != nil {
+			outcome = "error"
+		}
+		recordAgentRun(name, outcome, time.Since(started).Seconds())
 
 		if response == nil {
 			errMsg, _ := json.Marshal(map[string]any{

core/services/agentpool/metrics.go

@@ -0,0 +1,54 @@
+package agentpool
+
+import (
+	"context"
+	"sync"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// Prometheus metrics for agent chat runs. Operators need a scrape-friendly
+// signal for "are agent turns completing, erroring or getting cancelled,
+// and how long do they take" — log-derived counters proved brittle
+// (ANSI/timezone parsing, container-restart gaps). Chat() is the single
+// choke point of the local execution path, so instrumenting the response
+// handoff covers UI chats, API chats and connector-triggered asks alike.
+//
+// Lazily initialised on first record so the package works no matter when
+// (or whether) the Prometheus-backed global MeterProvider is installed —
+// same pattern as core/services/routing/pii.
+var (
+	agentMetricsOnce sync.Once
+	runsCounter      metric.Int64Counter
+	runSeconds       metric.Float64Histogram
+)
+
+func recordAgentRun(agent, outcome string, seconds float64) {
+	agentMetricsOnce.Do(func() {
+		meter := otel.Meter("github.com/mudler/LocalAI")
+		if c, err := meter.Int64Counter(
+			"localai_agent_runs_total",
+			metric.WithDescription("Agent chat runs, labeled by agent and outcome (completed|error|cancelled)"),
+		); err == nil {
+			runsCounter = c
+		}
+		if h, err := meter.Float64Histogram(
+			"localai_agent_run_seconds",
+			metric.WithDescription("Wall-clock duration of agent chat runs in seconds"),
+		); err == nil {
+			runSeconds = h
+		}
+	})
+	attrs := metric.WithAttributes(
+		attribute.String("agent", agent),
+		attribute.String("outcome", outcome),
+	)
+	if runsCounter != nil {
+		runsCounter.Add(context.Background(), 1, attrs)
+	}
+	if runSeconds != nil {
+		runSeconds.Record(context.Background(), seconds, attrs)
+	}
+}

docs/content/features/api-discovery.md

@@ -36,6 +36,7 @@ Returns the instance version, all available endpoint URLs (flat and categorized)
   "endpoints": {
     "chat_completions": "/v1/chat/completions",
     "models": "/v1/models",
+    "models_capabilities": "/v1/models/capabilities",
     "config_metadata": "/api/models/config-metadata",
     "instructions": "/api/instructions",
     "swagger": "/swagger/index.html"
@@ -123,6 +124,45 @@ Add `?format=json` to get a raw **OpenAPI fragment** (filtered Swagger spec with
 curl http://localhost:8080/api/instructions/config-management?format=json
 ```
 
+## Model Capabilities
+
+`GET /v1/models/capabilities`
+
+An additive, LocalAI-specific superset of `/v1/models`. It returns the same set of models but enriches each entry with the **capabilities** the model supports and the **input/output modalities** it accepts and produces. Use it to decide, before sending a request, whether a given model can take an image, audio, or video attachment directly — or whether the input needs converting/transcribing first.
+
+Because it is purely additive, clients that only understand `/v1/models` keep working unchanged; they simply never call this route.
+
+```bash
+curl http://localhost:8080/v1/models/capabilities
+```
+
+```json
+{
+  "object": "list",
+  "data": [
+    {
+      "id": "qwen2.5-omni",
+      "object": "model",
+      "capabilities": ["chat", "vision", "tools"],
+      "input_modalities": ["text", "image", "audio"],
+      "output_modalities": ["text"]
+    },
+    {
+      "id": "parakeet",
+      "object": "model",
+      "capabilities": ["transcript"],
+      "input_modalities": ["audio"],
+      "output_modalities": ["text"]
+    }
+  ]
+}
+```
+
+- **`capabilities`** — canonical usecase strings (e.g. `chat`, `vision`, `transcript`, `tts`, `embeddings`, `image`, `video`) plus the modifiers `tools` and `thinking`.
+- **`input_modalities` / `output_modalities`** — the subsets of `{text, image, audio, video}` the model accepts and produces. Audio and video *input* are derived from the model's multimodal limits (e.g. vLLM `limit_mm_per_prompt`), which no single usecase flag expresses — which is why this endpoint exists alongside the plain listing.
+
+The same query parameters as `/v1/models` are honored (`filter`, `excludeConfigured`), and the same per-user model allowlist is applied when authentication is enabled.
+
 ## Configuration Management APIs
 
 These endpoints let agents discover model configuration fields, read current settings, modify them, and estimate VRAM usage.

docs/content/whats-new.md

@@ -17,6 +17,7 @@ You can see the release notes [here](https://github.com/mudler/LocalAI/releases)
 - **May 2026**: [Speaker diarization](/features/audio-diarization/) — new `/v1/audio/diarization` endpoint returning "who spoke when" segments. Backed by `sherpa-onnx` (pyannote-3.0 + speaker embeddings + clustering) for pure diarization, and `vibevoice-cpp` for diarization bundled with long-form ASR. Supports `json` / `verbose_json` / `rttm` response formats.
 - **June 2026**: [Sound classification](/features/audio-classification/) — new `/v1/audio/classification` endpoint for audio tagging / sound-event classification, returning scored [AudioSet](https://research.google.com/audioset/) labels (baby cry, glass breaking, alarms, ...). Backed by [ced.cpp](https://github.com/mudler/ced.cpp), a 527-class AudioSet tagger ported to ggml.
 - **June 2026**: [PII analyze / redact API](/features/middleware/#analyze--redact-api) — the PII detection pipeline (NER + restricted-regex pattern tiers) is now a standalone service: `POST /api/pii/analyze` returns detected entity spans and `POST /api/pii/redact` returns the sanitised text (or `400 pii_blocked`), without routing a chat request through the middleware. Events gain an `origin` (`middleware` / `proxy` / `pii_analyze` / `pii_redact`) so `/api/pii/events` can be filtered by source.
+- **July 2026**: [Model capabilities endpoint](/features/api-discovery/#model-capabilities) — `GET /v1/models/capabilities`, an additive superset of `/v1/models` that reports each model's `capabilities` plus its `input_modalities` / `output_modalities` (`text` / `image` / `audio` / `video`). Lets clients route image/audio/video attachments to a model only when it can handle them; audio/video *input* is derived from the model's multimodal limits, which no single usecase flag expresses.
 - **June 2026**: Concurrent scoring and PII NER on llama.cpp — the `Score` (router classifier) and `TokenClassify` (PII NER) primitives now ride llama.cpp's server task queue instead of locking the context, so they run concurrently with chat/completion/embedding traffic and with each other. The `known_usecases` restriction that forced dedicated scorer/NER model configs on llama-cpp is lifted, repeated scoring calls reuse the prompt KV cache across candidates, and scoring inputs are no longer capped by the physical batch size.
 
 ## 2024 Highlights

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v4.5.6"
+  "version": "v4.6.0"
 }

gallery/index.yaml

@@ -3,22 +3,7 @@
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
     - https://huggingface.co/Jackrong/Qwopus3.6-35B-A3B-Coder-MTP-GGUF
-  description: |
-    # 🌟 Qwopus3.6-35B-A3B-v1
-
-    ## 💡 Base Model Overview
-
-    **Qwen3.6-35B-A3B** is an advanced hybrid sparse MoE (Mixture-of-Experts) model developed by Alibaba Cloud. It features 35B total parameters with only 3B active parameters per token, ensuring high inference efficiency. Architecturally, it combines Gated DeltaNet linear attention with standard gated attention layers, routing tokens across **256 experts**. It natively supports a massive **262k context window** and is specifically designed for high-performance agentic coding, deep reasoning, and multimodal tasks.
-
-    ## 🚀 Model Refinement & Logic Tuning （Qwopus3.6-35B-A3B-v1）
-
-    🪐**Qwopus3.6-35B-A3B-v1** is a reasoning-enhanced MoE (Mixture of Experts) model fine-tuned on top of **Qwen3.6-35B-A3B**.
-
-    ### 🛠 Training Strategy
-
-    The fine-tuning process for this model is structured into **three distinct stages of distributed SFT (Supervised Fine-Tuning)**, progressively scaling reasoning complexity and data diversity. This systematic approach ensures the model inherits the base MoE capabilities while sharpening its logic-handling depth.
-
-    ...
+  description: "# \U0001F31F Qwopus3.6-35B-A3B-v1\n\n## \U0001F4A1 Base Model Overview\n\n**Qwen3.6-35B-A3B** is an advanced hybrid sparse MoE (Mixture-of-Experts) model developed by Alibaba Cloud. It features 35B total parameters with only 3B active parameters per token, ensuring high inference efficiency. Architecturally, it combines Gated DeltaNet linear attention with standard gated attention layers, routing tokens across **256 experts**. It natively supports a massive **262k context window** and is specifically designed for high-performance agentic coding, deep reasoning, and multimodal tasks.\n\n## \U0001F680 Model Refinement & Logic Tuning （Qwopus3.6-35B-A3B-v1）\n\n\U0001FA90**Qwopus3.6-35B-A3B-v1** is a reasoning-enhanced MoE (Mixture of Experts) model fine-tuned on top of **Qwen3.6-35B-A3B**.\n\n### \U0001F6E0 Training Strategy\n\nThe fine-tuning process for this model is structured into **three distinct stages of distributed SFT (Supervised Fine-Tuning)**, progressively scaling reasoning complexity and data diversity. This systematic approach ensures the model inherits the base MoE capabilities while sharpening its logic-handling depth.\n\n...\n"
   license: "apache-2.0"
   tags:
     - llm
@@ -55,34 +40,7 @@
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
     - https://huggingface.co/protoLabsAI/Ornith-1.0-9B-MTP-GGUF
-  description: |
-    [](https://deep-reinforce.com/ornith.html)
-
-    # Ornith-1.0-9B
-
-    Aloha! 🌺 Today, we are releasing Ornith-1.0, a self-improving family of open-source models for agentic coding.
-
-    Highlights:
-
-      - **State-of-the-Art Coding Agents**: Available in 9B-Dense, 31B-Dense, 35B-MoE, and 397B-MoE (post-trained on top of Gemma 4 and Qwen 3.5), achieving state-of-the-art performance among open-source models of comparable size on coding benchmarks such as Terminal-Bench 2.1, SWE-Bench, NL2Repo and OpenClaw.
-      - **Self-Improving Training Framework**:  Ornith-1.0 employs RL to learn to generate not only solution rollouts, but also the scallfold that drive those rollouts. By jointly optimizing the scaffold and the resulting solution, the model  discovers better search trajectories and generates higher-quality solutions.
-      - **Licence**: MIT licensed, globally accessible, and free from regional limitations.
-
-    ## Ornith 1.0 9B
-
-    This model card documents **Ornith-1.0-9B**, the most lightweight member of the Ornith family, designed for efficient single-GPU deployment.
-
-    ### Benchmarks
-
-    Ornith-1.0-9B
-    Qwen3.5-9B
-    Qwen3.5-35B
-    Gemma4-12B
-    Gemma4-31B
-
-    Agentic Coding
-
-    ...
+  description: "[](https://deep-reinforce.com/ornith.html)\n\n# Ornith-1.0-9B\n\nAloha! \U0001F33A Today, we are releasing Ornith-1.0, a self-improving family of open-source models for agentic coding.\n\nHighlights:\n\n  - **State-of-the-Art Coding Agents**: Available in 9B-Dense, 31B-Dense, 35B-MoE, and 397B-MoE (post-trained on top of Gemma 4 and Qwen 3.5), achieving state-of-the-art performance among open-source models of comparable size on coding benchmarks such as Terminal-Bench 2.1, SWE-Bench, NL2Repo and OpenClaw.\n  - **Self-Improving Training Framework**:  Ornith-1.0 employs RL to learn to generate not only solution rollouts, but also the scallfold that drive those rollouts. By jointly optimizing the scaffold and the resulting solution, the model  discovers better search trajectories and generates higher-quality solutions.\n  - **Licence**: MIT licensed, globally accessible, and free from regional limitations.\n\n## Ornith 1.0 9B\n\nThis model card documents **Ornith-1.0-9B**, the most lightweight member of the Ornith family, designed for efficient single-GPU deployment.\n\n### Benchmarks\n\nOrnith-1.0-9B\nQwen3.5-9B\nQwen3.5-35B\nGemma4-12B\nGemma4-31B\n\nAgentic Coding\n\n...\n"
   license: "mit"
   tags:
     - llm
@@ -36066,7 +36024,7 @@
   files:
     - filename: parakeet-tdt-0.6b-ja.gguf
       uri: huggingface://cstr/parakeet-tdt-0.6b-ja-GGUF/parakeet-tdt-0.6b-ja.gguf
-      sha256: a9c43116b180b8a2ada2771ac829cf751b9e73adcbe69b7c8379593f9e5da31e
+      sha256: 374eb0132eebaec4df77a9631cbbeb03790be48a4a517f6cc8e8bdb38fe9a584
 - name: parakeet-tdt-1.1b-crispasr
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:
@@ -36525,7 +36483,7 @@
   files:
     - filename: vibevoice-realtime-0.5b-q4_k.gguf
       uri: huggingface://cstr/vibevoice-realtime-0.5b-GGUF/vibevoice-realtime-0.5b-q4_k.gguf
-      sha256: e3244986d8939a9a8f65701196efbfe3f8b81afd307b29f434fe259b9c411ef1
+      sha256: 483e1922a9077e3fc66b7947a4d6fee3dfd8edc30afde3410efa5bb386bc0392
 - name: chatterbox-tts-crispasr
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:

scripts/build/package-gpu-libs-rocm-data_test.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Regression test for scripts/build/package-gpu-libs.sh ROCm data bundling.
+#
+# Guards issue #10660: hipBLASLt (rocblaslt) resolves its TensileLibrary_lazy_gfx*.dat
+# kernel data relative to the bundled libhipblaslt.so. The packager copied the
+# rocblas/ data dir but not the hipblaslt/ data dir, so the bundled backend
+# fell back to slow generic kernels and logged
+#   rocblaslt error: Cannot read "TensileLibrary_lazy_gfx1201.dat": No such file or directory
+#
+# This test fabricates a fake ROCm tree containing both rocblas/ and hipblaslt/
+# tensile data, points the packager at it via ROCM_BASE_DIRS, and asserts BOTH
+# data directories are bundled into the target lib dir.
+set -euo pipefail
+
+CURDIR=$(dirname "$(realpath "$0")")
+SCRIPT="$CURDIR/package-gpu-libs.sh"
+
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Fabricate a fake ROCm install with both rocblas and hipblaslt tensile data.
+FAKE_ROCM="$WORK/opt/rocm"
+mkdir -p "$FAKE_ROCM/lib/rocblas/library"
+mkdir -p "$FAKE_ROCM/lib/hipblaslt/library"
+echo "fake rocblas tensile" > "$FAKE_ROCM/lib/rocblas/library/TensileLibrary_lazy_gfx1201.dat"
+echo "fake hipblaslt tensile" > "$FAKE_ROCM/lib/hipblaslt/library/TensileLibrary_lazy_gfx1201.dat"
+
+TARGET="$WORK/target"
+mkdir -p "$TARGET"
+
+# shellcheck source=/dev/null
+source "$SCRIPT" "$TARGET"
+
+# Point the data-dir copy at the fabricated tree instead of the real /opt/rocm,
+# then run the actual ROCm packager. This asserts package_rocm_libs itself
+# bundles BOTH data dirs, not just that the helper works in isolation.
+export BUILD_TYPE=hipblas
+export ROCM_BASE_DIRS="$FAKE_ROCM"
+package_rocm_libs
+
+fail=false
+if [ ! -e "$TARGET/rocblas/library/TensileLibrary_lazy_gfx1201.dat" ]; then
+    echo "FAIL: rocblas tensile data was NOT bundled"
+    fail=true
+fi
+if [ ! -e "$TARGET/hipblaslt/library/TensileLibrary_lazy_gfx1201.dat" ]; then
+    echo "FAIL: hipblaslt tensile data was NOT bundled (regression of #10660)"
+    fail=true
+fi
+
+if [ "$fail" = true ]; then
+    ls -R "$TARGET" || true
+    exit 1
+fi
+
+echo "PASS: rocblas and hipblaslt tensile data were both bundled"
+exit 0

scripts/build/package-gpu-libs.sh

@@ -224,6 +224,50 @@ package_cuda_libs() {
     echo "CUDA libraries packaged successfully"
 }
 
+# Copy a ROCm library data subdirectory (e.g. rocblas, hipblaslt) into the
+# bundled lib/ dir. These directories hold the TensileLibrary_*.dat GPU kernel
+# tuning files, which rocBLAS/hipBLASLt load at runtime *relative to their own
+# .so*. Since backends ship their own copies of libhipblaslt.so/librocblas.so
+# under lib/, the matching data dir must travel with them or the libs fall back
+# to slow generic kernels (rocblaslt error: Cannot read TensileLibrary_lazy_gfx*.dat;
+# see issue #10660).
+#
+# The ROCm search roots default to /opt/rocm{,-*} but can be overridden via the
+# ROCM_BASE_DIRS env var (space-separated), which keeps the copy unit-testable
+# without a real ROCm install.
+# Args: $1 = data subdir name found under <rocm-root>/lib{,64}/
+copy_rocm_data_dir() {
+    local data_name="$1"
+    # Single-line `local x=$(...)` on purpose: `local` masks the command
+    # substitution's exit status, which is 1 when nullglob is unset and would
+    # otherwise trip the script's `set -e`.
+    local old_nullglob=$(shopt -p nullglob)
+    shopt -s nullglob
+    local rocm_dirs
+    if [ -n "${ROCM_BASE_DIRS:-}" ]; then
+        # shellcheck disable=SC2206  # intentional word-split of the override
+        rocm_dirs=(${ROCM_BASE_DIRS})
+    else
+        rocm_dirs=(/opt/rocm /opt/rocm-*)
+    fi
+    eval "$old_nullglob"
+    local found=false
+    local rocm_base lib_subdir
+    for rocm_base in "${rocm_dirs[@]}"; do
+        for lib_subdir in lib lib64; do
+            if [ -d "$rocm_base/$lib_subdir/$data_name" ]; then
+                echo "Found $data_name data at $rocm_base/$lib_subdir/$data_name"
+                mkdir -p "$TARGET_LIB_DIR/$data_name"
+                cp -arfL "$rocm_base/$lib_subdir/$data_name/"* "$TARGET_LIB_DIR/$data_name/" || echo "WARNING: Failed to copy $data_name data from $rocm_base/$lib_subdir/$data_name"
+                found=true
+            fi
+        done
+    done
+    if [ "$found" = false ]; then
+        echo "WARNING: No $data_name library data found in ${ROCM_BASE_DIRS:-/opt/rocm*}/lib{,64}/$data_name"
+    fi
+}
+
 # Package AMD ROCm/HIPBlas libraries
 package_rocm_libs() {
     echo "Packaging ROCm/HIPBlas libraries for BUILD_TYPE=${BUILD_TYPE}..."
@@ -267,27 +311,16 @@ package_rocm_libs() {
         fi
     done
 
-    # Copy rocblas library data (tuning files, TensileLibrary, etc.)
-    local old_nullglob=$(shopt -p nullglob)
-    shopt -s nullglob
-    local rocm_dirs=(/opt/rocm /opt/rocm-*)
-    eval "$old_nullglob"
-    local rocblas_found=false
-    for rocm_base in "${rocm_dirs[@]}"; do
-        for lib_subdir in lib lib64; do
-            if [ -d "$rocm_base/$lib_subdir/rocblas" ]; then
-                echo "Found rocblas data at $rocm_base/$lib_subdir/rocblas"
-                mkdir -p "$TARGET_LIB_DIR/rocblas"
-                cp -arfL "$rocm_base/$lib_subdir/rocblas/"* "$TARGET_LIB_DIR/rocblas/" || echo "WARNING: Failed to copy rocblas data from $rocm_base/$lib_subdir/rocblas"
-                rocblas_found=true
-            fi
-        done
-    done
-    if [ "$rocblas_found" = false ]; then
-        echo "WARNING: No rocblas library data found in /opt/rocm*/lib{,64}/rocblas"
-    fi
+    # Copy rocBLAS and hipBLASLt kernel data (TensileLibrary_*.dat tuning files)
+    # so the bundled libs find their per-arch kernels at runtime instead of
+    # falling back to slow generic code (see copy_rocm_data_dir / issue #10660).
+    copy_rocm_data_dir rocblas
+    copy_rocm_data_dir hipblaslt
 
     # Copy libomp from LLVM (required for ROCm)
+    # Single-line `local x=$(...)` on purpose: masks shopt -p's nonzero exit
+    # (nullglob unset) so it doesn't trip `set -e`.
+    local old_nullglob=$(shopt -p nullglob)
     shopt -s nullglob
     local omp_libs=(/opt/rocm*/lib/llvm/lib/libomp.so*)
     eval "$old_nullglob"
@@ -477,6 +510,7 @@ export -f copy_libs_glob
 export -f is_core_lib
 export -f copy_elf_deps
 export -f sweep_transitive_deps
+export -f copy_rocm_data_dir
 export -f package_cuda_libs
 export -f package_rocm_libs
 export -f package_intel_libs

swagger/docs.go

@@ -2728,6 +2728,22 @@ const docTemplate = `{
                 }
             }
         },
+        "/v1/models/capabilities": {
+            "get": {
+                "tags": [
+                    "models"
+                ],
+                "summary": "List available models enriched with capabilities and input/output modalities.",
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.ModelCapabilitiesResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/v1/rerank": {
             "post": {
                 "tags": [
@@ -5182,6 +5198,52 @@ const docTemplate = `{
                 }
             }
         },
+        "schema.ModelCapabilities": {
+            "type": "object",
+            "properties": {
+                "capabilities": {
+                    "description": "Capabilities are canonical usecase strings (e.g. chat, vision, transcript,\ntts, embeddings, image, video) plus the modifiers \"tools\" and \"thinking\".",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "input_modalities": {
+                    "description": "InputModalities is the subset of {text,image,audio,video} the model accepts.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                },
+                "output_modalities": {
+                    "description": "OutputModalities is the subset of {text,image,audio,video} the model produces.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "schema.ModelCapabilitiesResponse": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ModelCapabilities"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                }
+            }
+        },
         "schema.ModelLoadRequest": {
             "type": "object",
             "properties": {

swagger/swagger.json

@@ -2725,6 +2725,22 @@
                 }
             }
         },
+        "/v1/models/capabilities": {
+            "get": {
+                "tags": [
+                    "models"
+                ],
+                "summary": "List available models enriched with capabilities and input/output modalities.",
+                "responses": {
+                    "200": {
+                        "description": "Response",
+                        "schema": {
+                            "$ref": "#/definitions/schema.ModelCapabilitiesResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/v1/rerank": {
             "post": {
                 "tags": [
@@ -5179,6 +5195,52 @@
                 }
             }
         },
+        "schema.ModelCapabilities": {
+            "type": "object",
+            "properties": {
+                "capabilities": {
+                    "description": "Capabilities are canonical usecase strings (e.g. chat, vision, transcript,\ntts, embeddings, image, video) plus the modifiers \"tools\" and \"thinking\".",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "id": {
+                    "type": "string"
+                },
+                "input_modalities": {
+                    "description": "InputModalities is the subset of {text,image,audio,video} the model accepts.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                },
+                "output_modalities": {
+                    "description": "OutputModalities is the subset of {text,image,audio,video} the model produces.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "schema.ModelCapabilitiesResponse": {
+            "type": "object",
+            "properties": {
+                "data": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/schema.ModelCapabilities"
+                    }
+                },
+                "object": {
+                    "type": "string"
+                }
+            }
+        },
         "schema.ModelLoadRequest": {
             "type": "object",
             "properties": {

swagger/swagger.yaml

@@ -1362,6 +1362,41 @@ definitions:
           $ref: '#/definitions/schema.ToolCall'
         type: array
     type: object
+  schema.ModelCapabilities:
+    properties:
+      capabilities:
+        description: |-
+          Capabilities are canonical usecase strings (e.g. chat, vision, transcript,
+          tts, embeddings, image, video) plus the modifiers "tools" and "thinking".
+        items:
+          type: string
+        type: array
+      id:
+        type: string
+      input_modalities:
+        description: InputModalities is the subset of {text,image,audio,video} the
+          model accepts.
+        items:
+          type: string
+        type: array
+      object:
+        type: string
+      output_modalities:
+        description: OutputModalities is the subset of {text,image,audio,video} the
+          model produces.
+        items:
+          type: string
+        type: array
+    type: object
+  schema.ModelCapabilitiesResponse:
+    properties:
+      data:
+        items:
+          $ref: '#/definitions/schema.ModelCapabilities'
+        type: array
+      object:
+        type: string
+    type: object
   schema.ModelLoadRequest:
     properties:
       model:
@@ -4358,6 +4393,16 @@ paths:
       summary: List and describe the various models available in the API.
       tags:
       - models
+  /v1/models/capabilities:
+    get:
+      responses:
+        "200":
+          description: Response
+          schema:
+            $ref: '#/definitions/schema.ModelCapabilitiesResponse'
+      summary: List available models enriched with capabilities and input/output modalities.
+      tags:
+      - models
   /v1/rerank:
     post:
       parameters: