fix(config): cap auto-derived context to fit VRAM

When a model is imported without an explicit context_size, the GGUF
importer defaulted the model's context to its full trained window
(n_ctx_train). For long-context models (128k / 256k / 1M) that KV cache
cannot fit a consumer GPU, so the backend aborts on load (exitCode=-1)
even though the model file is perfectly fine. Reproduced live:
gemma-4-26b-a4b-it-qat-q4_0 defaulted to context=262144 and
qwythos-9b-claude-mythos-5-1m to 1048576, both aborting on a 20 GB card.

Instead of chasing the trained max, auto-derive a conservative default:
min(trainedMax, DefaultAutoContextSize=8192). A small model keeps its
trained window; a long-context model caps at 8k and users opt into more
via context_size. This cap applies always, including CPU / unknown-VRAM
hosts, so it never regresses those paths.

Per-device VRAM is used only as a DOWNWARD safety: when a per-device
ceiling is detected (xsysinfo.MinPerGPUVRAM) and even the 8k cap would
not fit it with headroom, step down through candidate contexts to the
largest that fits, floored at DefaultContextSize. When VRAM is unknown
(0) or no GPU is detected we do NOT clamp — the bug is GPU OOM and the
8k cap is already safe, so detection gaps must not shrink the window.

The footprint estimate reuses gpustack/gguf-parser-go's
EstimateLLaMACppRun at a given context with all layers offloaded, taking
the per-device NonUMA VRAM figure. The estimate and VRAM detection are
package vars so tests inject deterministic values. Explicit context_size
always wins (guessGGUFFromFile only acts when it is nil).

Assisted-by: Claude:claude-opus-4-8 [golangci-lint go-test]

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=665892536dfb1b7532161e3182304bd35c33e768
+LLAMA_VERSION?=2da668617612d2df773f966e3b0ee22dc2beef7b
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/go/omnivoice-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # omnivoice.cpp version
 OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp
-OMNIVOICE_VERSION?=0f37401bebe9b20c0160a888e592108fc1d17607
+OMNIVOICE_VERSION?=daedb763fd442e0916eb130a479fdd74947291c0
 SO_TARGET?=libgomnivoicecpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/qwen3-tts-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # qwentts.cpp version
 QWEN3TTS_REPO?=https://github.com/ServeurpersoCom/qwentts.cpp
-QWEN3TTS_CPP_VERSION?=9dbe7ea26a01b30fccb117ae5e86807c1dc23d42
+QWEN3TTS_CPP_VERSION?=73fe0c67bbf0898ba2999535e0680a02a7f8537d
 SO_TARGET?=libgoqwen3ttscpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

core/backend/options.go

@@ -223,24 +223,13 @@ func EffectiveContextSize(c config.ModelConfig) int {
 	return DefaultContextSize
 }
 
-// localGPU resolves the device that will run the model, for single-pass batch
-// sizing. It is a package var so tests inject a deterministic device; production
-// reads config.LocalGPU, whose detection is sync.Once-cached in xsysinfo — so the
-// per-request call from the router's prompt trimmer (modelTokenTrim) stays cheap.
-var localGPU = config.LocalGPU
-
 // EffectiveBatchSize is the single-decode batch the backend will run with.
 // Score, embedding and rerank all process the whole input in one pass: score
 // decodes prompt+candidate (asserts n_tokens <= n_batch), and embedding/rerank
-// pool over the full sequence in one physical batch (n_ubatch). Ideally the batch
-// covers the whole context so any input that fits the context fits one pass,
+// pool over the full sequence in one physical batch (n_ubatch). So the batch
+// is sized to the context — anything that fits the context fits one pass,
 // avoiding both the GGML_ASSERT crash and the "input is too large to process"
-// error — BUT a full ctx-sized n_ubatch makes the per-device CUDA compute buffer
-// multi-GiB (it scales ~ n_ubatch * n_ctx and can't be split across GPUs), so a
-// large-context embedding model aborts on load with free VRAM to spare (#10485).
-// So we cap the batch to the largest that fits the per-device VRAM headroom; an
-// input longer than that cap is the accepted tradeoff (it can't be pooled in one
-// pass, but the load no longer OOMs). Explicit `batch:` always wins.
+// error. Explicit `batch:` always wins.
 func EffectiveBatchSize(c config.ModelConfig) int {
 	if c.Batch != 0 {
 		return c.Batch
@@ -249,7 +238,7 @@ func EffectiveBatchSize(c config.ModelConfig) int {
 		c.HasUsecases(config.FLAG_EMBEDDINGS) ||
 		c.HasUsecases(config.FLAG_RERANK)
 	if ctx := EffectiveContextSize(c); singlePass && ctx > DefaultBatchSize {
-		return config.SinglePassBatchForContext(localGPU(), ctx)
+		return ctx
 	}
 	return DefaultBatchSize
 }

core/backend/options_internal_test.go

@@ -103,19 +103,6 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	threads := 1
 	ctx := 4096
 
-	// The single-pass batch is now VRAM-aware, so inject a deterministic GPU with
-	// ample per-device VRAM: at these small contexts the compute buffer fits
-	// easily, so EffectiveBatchSize returns the full context (the pre-#10485
-	// behaviour these cases assert). Without injection the value would depend on
-	// the CI host's real (often unknown) VRAM.
-	const gib = uint64(1) << 30
-	var origLocalGPU func() config.GPU
-	BeforeEach(func() {
-		origLocalGPU = localGPU
-		localGPU = func() config.GPU { return config.GPU{VRAM: 119 * gib} }
-	})
-	AfterEach(func() { localGPU = origLocalGPU })
-
 	It("defaults to 512 for an ordinary model", func() {
 		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
 		opts := grpcModelOpts(cfg, "/tmp/models")
@@ -175,58 +162,6 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	})
 })
 
-// Guards the VRAM-aware cap on the single-pass (embedding/score/rerank) batch:
-// a large context must not turn n_ubatch into a multi-GiB compute buffer that
-// aborts the load on a device with free VRAM (issue #10485). The GPU is injected
-// via the localGPU package var so the cap is deterministic without a real device.
-var _ = Describe("EffectiveBatchSize VRAM cap", func() {
-	const gib = uint64(1) << 30
-	embeddings := config.FLAG_EMBEDDINGS
-	threads := 1
-
-	var origLocalGPU func() config.GPU
-	BeforeEach(func() { origLocalGPU = localGPU })
-	AfterEach(func() { localGPU = origLocalGPU })
-
-	singlePassCfg := func(ctx int) config.ModelConfig {
-		return config.ModelConfig{
-			Threads:       &threads,
-			LLMConfig:     config.LLMConfig{ContextSize: &ctx},
-			KnownUsecases: &embeddings,
-		}
-	}
-
-	It("caps a large embedding context to a batch below the context but at least the default", func() {
-		// Reproduces qwen3-embedding-4b: context 40960 on a modest 20 GiB card.
-		// Full-context n_ubatch=40960 aborts; the cap must fit the VRAM headroom.
-		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
-		batch := EffectiveBatchSize(singlePassCfg(40960))
-		Expect(batch).To(BeNumerically(">=", DefaultBatchSize))
-		Expect(batch).To(BeNumerically("<", 40960))
-	})
-
-	It("keeps an explicit batch even with a large context and small VRAM", func() {
-		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
-		cfg := singlePassCfg(40960)
-		cfg.Batch = 512
-		Expect(EffectiveBatchSize(cfg)).To(Equal(512))
-	})
-
-	It("stays conservative (default, not the context) when per-device VRAM is unknown", func() {
-		// A detection gap (VRAM 0) must not fall back to the unbounded context —
-		// that's exactly what would OOM the load.
-		localGPU = func() config.GPU { return config.GPU{VRAM: 0} }
-		Expect(EffectiveBatchSize(singlePassCfg(40960))).To(Equal(DefaultBatchSize))
-	})
-
-	It("returns the default batch for a non-single-pass model regardless of VRAM", func() {
-		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
-		ctx := 40960
-		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
-		Expect(EffectiveBatchSize(cfg)).To(Equal(DefaultBatchSize))
-	})
-})
-
 // Guards the generic chat_template_kwargs forwarding: the model config map plus any
 // per-request metadata overrides are merged, coerced, and serialised into the
 // backend metadata blob that llama.cpp reads. Client metadata also overrides the

core/config/context_fit.go

@@ -0,0 +1,144 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/xsysinfo"
+	"github.com/mudler/xlog"
+)
+
+// contextFitHeadroomDivisor reserves a slice of per-device VRAM as headroom when
+// deciding whether an auto-derived context fits. The gguf-parser footprint
+// already covers weights + KV + compute buffer, but a live load also pays for
+// allocator fragmentation, the CUDA/HIP context, and whatever else shares the
+// card, so we require the estimate to leave at least 1/divisor of the device
+// free. /5 (~20% headroom) mirrors the SWA full-cache gate's margin.
+const contextFitHeadroomDivisor = 5
+
+// contextFitCandidates is the descending set of context windows tried when the
+// DefaultAutoContextSize cap itself does not fit per-device VRAM. Only the rare
+// big-model-on-tiny-card case reaches this walk; it is capped at the base
+// choice and floored at DefaultContextSize, and returns the first (largest)
+// candidate that fits.
+var contextFitCandidates = []int{8192, 6144, 4096}
+
+// perDeviceVRAM reports the smallest per-GPU VRAM ceiling in bytes (0 = unknown
+// or no GPU). It is a package var so tests can inject a deterministic value —
+// detection does a live GPU probe. Per-device (not summed) is the right budget:
+// with all layers offloaded to a single device the whole footprint must fit that
+// one card, and a multi-GPU host is bounded by its smallest card. This mirrors
+// localGPU's use of MinPerGPUVRAM in hardware_defaults.go.
+var perDeviceVRAM = func() uint64 {
+	v, _ := xsysinfo.MinPerGPUVRAM()
+	return v
+}
+
+// estimateContextVRAM returns the estimated per-device VRAM footprint (bytes) of
+// running f fully offloaded at ctx tokens — weights + KV cache + compute buffer.
+// It returns 0 when it cannot produce an estimate (nil file, no tensors, or a
+// parser panic), which the caller treats as "cannot confirm a smaller fit" and
+// so keeps the conservative cap rather than clamping on a bogus number. It is a
+// package var so tests can stub it (a fabricated GGUF carries no tensors and
+// estimates to ~0).
+var estimateContextVRAM = func(f *gguf.GGUFFile, ctx int) (footprint uint64) {
+	if f == nil {
+		return 0
+	}
+	if ctx <= 0 {
+		ctx = DefaultContextSize
+	}
+	// The gguf-parser estimator panics on degenerate / partially-parsed GGUFs;
+	// treat any failure as "unknown" so config loading never crashes on a model
+	// the parser mis-handles.
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Debug("[context_fit] per-device VRAM estimate failed; treating as unknown", "error", r)
+			footprint = 0
+		}
+	}()
+	// Offload all layers (LocalAI's DefaultNGPULayers default; the estimator
+	// clamps to the model's block count) so the estimate reflects a fully
+	// GPU-resident model. NonUMA is the discrete-GPU figure (larger than the UMA
+	// one), which keeps the fit check conservative on unified-memory hosts — they
+	// have ample memory to clear it anyway.
+	est := f.EstimateLLaMACppRun(
+		gguf.WithLLaMACppContextSize(int32(ctx)),
+		gguf.WithLLaMACppOffloadLayers(uint64(DefaultNGPULayers)),
+	)
+	sum := est.Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return 0
+	}
+	var total uint64
+	for _, v := range sum.Items[0].VRAMs {
+		total += uint64(v.NonUMA)
+	}
+	return total
+}
+
+// contextFitsVRAM reports whether an estimated footprint fits a per-device VRAM
+// ceiling with headroom (VRAM must exceed the footprint by ~1/divisor). Unknown
+// inputs (0) are treated as "cannot confirm" so a detection or estimate gap does
+// not clamp the context.
+func contextFitsVRAM(footprint, vram uint64) bool {
+	if footprint == 0 || vram == 0 {
+		return false
+	}
+	return vram >= footprint+footprint/contextFitHeadroomDivisor
+}
+
+// autoContextSize picks the default context to use for f when the user did not
+// set context_size. The choice is deliberately conservative, NOT
+// VRAM-maximizing:
+//
+//  1. Base cap: min(trainedMax, DefaultAutoContextSize). A small model keeps its
+//     trained window; a long-context model (128k / 256k / 1M) is capped so its
+//     KV cache does not default to a size no consumer GPU can hold. This applies
+//     always, including CPU / unknown-VRAM hosts.
+//  2. VRAM is only a downward safety: when a per-device VRAM ceiling IS detected
+//     and even the base cap would not fit it (with headroom), step down through
+//     contextFitCandidates to the largest window that fits, floored at
+//     DefaultContextSize. When VRAM is unknown we skip this — the base cap is
+//     already safe and we must not regress CPU / detection-gap hosts.
+//
+// trainedMax <= 0 means the estimate yielded nothing usable; the caller keeps
+// its existing DefaultContextSize fallback in that case, so this is only called
+// with a positive trainedMax.
+func autoContextSize(f *gguf.GGUFFile, trainedMax int) int {
+	chosen := trainedMax
+	if chosen > DefaultAutoContextSize {
+		chosen = DefaultAutoContextSize
+	}
+
+	vram := perDeviceVRAM()
+	if vram == 0 {
+		// No per-device VRAM detected (CPU-only, unified memory reporting nothing,
+		// or a detection gap). The bug is GPU OOM-on-load, so with no GPU budget to
+		// reason about we must not clamp — the base cap already bounds long-context
+		// models.
+		return chosen
+	}
+
+	if contextFitsVRAM(estimateContextVRAM(f, chosen), vram) {
+		return chosen
+	}
+
+	// The base cap does not fit this card. Walk candidates downward and take the
+	// largest that fits, never below DefaultContextSize.
+	for _, cand := range contextFitCandidates {
+		if cand > chosen || cand < DefaultContextSize {
+			continue
+		}
+		if contextFitsVRAM(estimateContextVRAM(f, cand), vram) {
+			xlog.Debug("[context_fit] capped auto context to fit per-device VRAM",
+				"context", cand, "base_cap", chosen, "vram_gib", vram>>30)
+			return cand
+		}
+	}
+
+	// Nothing fit (an unusually large model on a tiny card): fall back to the
+	// floor. The backend still clamps n_gpu_layers to what fits, so a partial
+	// offload can keep the model loadable rather than aborting outright.
+	xlog.Debug("[context_fit] no candidate context fit per-device VRAM; using floor",
+		"context", DefaultContextSize, "base_cap", chosen, "vram_gib", vram>>30)
+	return DefaultContextSize
+}

core/config/context_fit_internal_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These specs exercise the auto-derived default context. The detection seams
+// (perDeviceVRAM, estimateContextVRAM) are package vars so a deterministic VRAM
+// ceiling and footprint can be injected without a real GPU or model file — the
+// same pattern hardware_defaults_internal_test.go uses for localGPU.
+var _ = Describe("Auto-derived default context (VRAM-aware cap)", func() {
+	const gib = uint64(1) << 30
+
+	var (
+		origVRAM     func() uint64
+		origEstimate func(f *gguf.GGUFFile, ctx int) uint64
+	)
+
+	BeforeEach(func() {
+		origVRAM = perDeviceVRAM
+		origEstimate = estimateContextVRAM
+	})
+	AfterEach(func() {
+		perDeviceVRAM = origVRAM
+		estimateContextVRAM = origEstimate
+	})
+
+	Context("autoContextSize", func() {
+		It("caps a long-context model at DefaultAutoContextSize when VRAM is ample", func() {
+			// 1M-context model on an 80 GiB card: we do NOT chase the trained max,
+			// we keep the conservative 8k cap (users opt into more via context_size).
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib } // trivially fits
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+
+		It("keeps a small model's trained window instead of inflating it", func() {
+			// trained 4096 < 8192: min() keeps 4096, it is not raised to the cap.
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib }
+			Expect(autoContextSize(nil, 4096)).To(Equal(4096))
+		})
+
+		It("steps below the cap when even 8k would not fit a tiny card", func() {
+			// A large model on a 2 GiB card where the 8k footprint overflows but a
+			// smaller context fits: choose the largest that fits, never below the
+			// floor. Footprint grows with context so the walk finds a fit.
+			perDeviceVRAM = func() uint64 { return 2 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, ctx int) uint64 {
+				return gib + uint64(ctx)*100000
+			}
+			chosen := autoContextSize(nil, 1048576)
+			Expect(chosen).To(BeNumerically("<", DefaultAutoContextSize))
+			Expect(chosen).To(BeNumerically(">=", DefaultContextSize))
+			// The chosen context's footprint must actually fit the card with headroom.
+			Expect(contextFitsVRAM(estimateContextVRAM(nil, chosen), 2*gib)).To(BeTrue())
+		})
+
+		It("falls back to the floor when nothing fits", func() {
+			// Even DefaultContextSize does not fit: return the floor and let the
+			// backend clamp n_gpu_layers to what it can (partial offload) rather
+			// than defaulting to a window guaranteed to abort.
+			perDeviceVRAM = func() uint64 { return 1 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 100 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultContextSize))
+		})
+
+		It("does not clamp when per-device VRAM is unknown", func() {
+			// CPU-only / detection gap: no GPU budget to reason about, so we must
+			// not regress — keep the conservative base cap regardless of estimate.
+			perDeviceVRAM = func() uint64 { return 0 }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 999 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+	})
+
+	Context("guessGGUFFromFile", func() {
+		It("never overrides an explicitly configured context_size", func() {
+			// A fabricated GGUF is enough: the context branch is skipped entirely
+			// when the user pinned context_size, so the estimate is never consulted.
+			explicit := 262144
+			cfg := &ModelConfig{LLMConfig: LLMConfig{ContextSize: &explicit}}
+			f := &gguf.GGUFFile{
+				Header: gguf.GGUFHeader{
+					MetadataKV: gguf.GGUFMetadataKVs{
+						{
+							Key:       "general.architecture",
+							ValueType: gguf.GGUFMetadataValueTypeString,
+							Value:     "llama",
+						},
+					},
+				},
+			}
+			guessGGUFFromFile(cfg, f, 0)
+			Expect(cfg.ContextSize).ToNot(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(262144))
+		})
+	})
+})

core/config/defaults.go

@@ -18,6 +18,18 @@ const (
 	// safe default beats a tiny, surprising window that truncates real prompts.
 	DefaultContextSize = 4096
 
+	// DefaultAutoContextSize caps the context we auto-derive from a GGUF when the
+	// user did not set context_size. The GGUF importer used to default a model's
+	// context to its full trained window (n_ctx_train). For long-context models
+	// (128k / 256k / 1M) that KV cache cannot fit a consumer GPU and the backend
+	// aborts on load (exitCode=-1) even though the model file is fine. So instead
+	// of shooting for the trained max, we keep a modest default: a small model
+	// (trained < this) keeps its trained window, while a long-context model caps
+	// here. Users who want the full window raise context_size explicitly. This is
+	// a conservative default, not a VRAM-maximizing one — VRAM is only used to
+	// step further DOWN when even this cap would not fit (see context_fit.go).
+	DefaultAutoContextSize = 8192
+
 	// DefaultNGPULayers means "offload all layers"; the backend (fit_params)
 	// clamps to what actually fits in device memory.
 	DefaultNGPULayers = 99999999

core/config/gguf.go

@@ -28,9 +28,14 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 
 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
 	if defaultCtx == 0 && cfg.ContextSize == nil {
-		ctxSize := f.EstimateLLaMACppRun().ContextSize
-		if ctxSize > 0 {
-			cSize := int(ctxSize)
+		// trainedMax is the model's full trained context window (n_ctx_train).
+		// Defaulting a model to it unbounded is what OOMs long-context models at
+		// load: a 128k / 256k / 1M KV cache cannot fit a consumer GPU and the
+		// backend aborts (exitCode=-1). autoContextSize instead caps to a modest
+		// default and only steps below it when detected per-device VRAM demands.
+		trainedMax := int(f.EstimateLLaMACppRun().ContextSize)
+		if trainedMax > 0 {
+			cSize := autoContextSize(f, trainedMax)
 			cfg.ContextSize = &cSize
 		} else {
 			defaultCtx = DefaultContextSize

core/config/hardware_defaults.go

@@ -149,47 +149,6 @@ func largeContextForDevice(g GPU, ctx int) bool {
 	return extra > g.VRAM/blackwellBatchHeadroomDivisor
 }
 
-// SinglePassBatchForContext caps the physical batch (n_batch / n_ubatch) for a
-// single-pass load — embedding, score and rerank all decode/pool the whole input
-// in ONE physical batch, so they want a batch >= the input length to avoid the
-// GGML_ASSERT(n_tokens <= n_batch) abort and the "input is too large to process"
-// error. The naive choice is batch == context, but n_ubatch == context turns the
-// per-device CUDA compute buffer (which scales ~ n_ubatch * n_ctx and is NOT
-// split across GPUs) into multi-GiB of scratch that must fit on a SINGLE card, so
-// a large-context embedding model aborts on load (exitCode=-1) even with plenty
-// of free VRAM — the same #10485 root cause the Blackwell batch boost guards
-// against, which the single-pass path previously bypassed entirely.
-//
-// So instead of the full context we return the LARGEST batch whose compute buffer
-// fits the per-device VRAM headroom (VRAM / blackwellBatchHeadroomDivisor),
-// clamped to [DefaultPhysicalBatch, ctx]. The tradeoff: an input longer than the
-// returned cap can no longer be pooled in a single pass — but a batch that OOMs
-// the device processes nothing at all.
-//
-// g.VRAM must be the PER-DEVICE ceiling (smallest device on a multi-GPU host).
-// VRAM 0 (unknown) stays conservative and returns DefaultPhysicalBatch rather
-// than the unbounded context, so a detection gap can't OOM the load.
-func SinglePassBatchForContext(g GPU, ctx int) int {
-	if ctx <= DefaultPhysicalBatch {
-		return DefaultPhysicalBatch
-	}
-	if g.VRAM == 0 {
-		return DefaultPhysicalBatch
-	}
-	perBatchCell := uint64(ctx) * computeBufferBytesPerCell
-	if perBatchCell == 0 {
-		return DefaultPhysicalBatch
-	}
-	batchCap := int(g.VRAM / blackwellBatchHeadroomDivisor / perBatchCell)
-	if batchCap < DefaultPhysicalBatch {
-		return DefaultPhysicalBatch
-	}
-	if batchCap > ctx {
-		return ctx
-	}
-	return batchCap
-}
-
 // IsManagedPhysicalBatch reports whether n is a value PhysicalBatch assigns.
 // Callers that re-tune a value chosen by an upstream host (the distributed
 // router correcting the frontend's guess) use this to avoid clobbering an
@@ -295,14 +254,6 @@ var localGPU = func() GPU {
 	}
 }
 
-// LocalGPU exposes the locally-detected device descriptor to other packages
-// (e.g. core/backend's single-pass batch sizing) so they resolve the same
-// per-device VRAM this package's heuristics reason about. It goes through the
-// injectable localGPU var, so a config-package test seam also affects callers.
-func LocalGPU() GPU {
-	return localGPU()
-}
-
 // ApplyHardwareDefaults fills ModelConfig values that depend on the target GPU
 // and were left unset by the user. Currently: a larger physical batch on
 // Blackwell. Explicit config always wins (we only touch zero values).

core/config/hardware_defaults_internal_test.go

@@ -46,38 +46,3 @@ var _ = Describe("SetDefaults hardware defaults (single-instance)", func() {
 		Expect(cfg.Batch).To(Equal(1024))
 	})
 })
-
-// SinglePassBatchForContext is the VRAM-aware cap for the single-pass
-// (embedding/score/rerank) batch — the compute buffer scales ~ n_ubatch * n_ctx
-// and must fit a single device, so a large context can't take the full context
-// as its batch (issue #10485).
-var _ = Describe("SinglePassBatchForContext", func() {
-	const gib = uint64(1) << 30
-
-	It("returns the default when the context is at or below the default batch", func() {
-		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, DefaultPhysicalBatch)).To(Equal(DefaultPhysicalBatch))
-		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, 256)).To(Equal(DefaultPhysicalBatch))
-	})
-
-	It("returns the full context when the compute buffer fits ample VRAM", func() {
-		// 4096 ctx on 119 GiB: the compute buffer is tiny, so the batch covers
-		// the whole context (single-pass pooling in one physical batch).
-		Expect(SinglePassBatchForContext(GPU{VRAM: 119 * gib}, 4096)).To(Equal(4096))
-	})
-
-	It("caps below the context when a large context would overflow the VRAM headroom", func() {
-		batch := SinglePassBatchForContext(GPU{VRAM: 20 * gib}, 40960)
-		Expect(batch).To(BeNumerically(">=", DefaultPhysicalBatch))
-		Expect(batch).To(BeNumerically("<", 40960))
-		// The compute buffer for the capped batch must fit VRAM/headroom.
-		Expect(uint64(batch) * 40960 * computeBufferBytesPerCell).To(BeNumerically("<=", (20*gib)/blackwellBatchHeadroomDivisor))
-	})
-
-	It("never caps below the default batch even when VRAM is very tight", func() {
-		Expect(SinglePassBatchForContext(GPU{VRAM: 1 * gib}, 40960)).To(Equal(DefaultPhysicalBatch))
-	})
-
-	It("stays conservative (default) when per-device VRAM is unknown", func() {
-		Expect(SinglePassBatchForContext(GPU{VRAM: 0}, 40960)).To(Equal(DefaultPhysicalBatch))
-	})
-})

core/services/agentpool/agent_pool.go

@@ -426,7 +426,15 @@ func (s *AgentPoolService) Chat(name, message string) (string, error) {
 
 	// Process asynchronously
 	go func() {
+		started := time.Now()
 		response := ag.Ask(coreTypes.WithText(message))
+		outcome := "completed"
+		if response == nil {
+			outcome = "cancelled"
+		} else if response.Error != nil {
+			outcome = "error"
+		}
+		recordAgentRun(name, outcome, time.Since(started).Seconds())
 
 		if response == nil {
 			errMsg, _ := json.Marshal(map[string]any{

core/services/agentpool/metrics.go

@@ -0,0 +1,54 @@
+package agentpool
+
+import (
+	"context"
+	"sync"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// Prometheus metrics for agent chat runs. Operators need a scrape-friendly
+// signal for "are agent turns completing, erroring or getting cancelled,
+// and how long do they take" — log-derived counters proved brittle
+// (ANSI/timezone parsing, container-restart gaps). Chat() is the single
+// choke point of the local execution path, so instrumenting the response
+// handoff covers UI chats, API chats and connector-triggered asks alike.
+//
+// Lazily initialised on first record so the package works no matter when
+// (or whether) the Prometheus-backed global MeterProvider is installed —
+// same pattern as core/services/routing/pii.
+var (
+	agentMetricsOnce sync.Once
+	runsCounter      metric.Int64Counter
+	runSeconds       metric.Float64Histogram
+)
+
+func recordAgentRun(agent, outcome string, seconds float64) {
+	agentMetricsOnce.Do(func() {
+		meter := otel.Meter("github.com/mudler/LocalAI")
+		if c, err := meter.Int64Counter(
+			"localai_agent_runs_total",
+			metric.WithDescription("Agent chat runs, labeled by agent and outcome (completed|error|cancelled)"),
+		); err == nil {
+			runsCounter = c
+		}
+		if h, err := meter.Float64Histogram(
+			"localai_agent_run_seconds",
+			metric.WithDescription("Wall-clock duration of agent chat runs in seconds"),
+		); err == nil {
+			runSeconds = h
+		}
+	})
+	attrs := metric.WithAttributes(
+		attribute.String("agent", agent),
+		attribute.String("outcome", outcome),
+	)
+	if runsCounter != nil {
+		runsCounter.Add(context.Background(), 1, attrs)
+	}
+	if runSeconds != nil {
+		runSeconds.Record(context.Background(), seconds, attrs)
+	}
+}