fix(router): production-ready request router + auto-size batch for embedding/rerank (#10104)

* fix(router): score classifier production-readiness

Conversation trimming runs through the classifier model's chat template
and trims by exact token count, sized to the model's n_batch which is
now scaled to context so long probes can't crash the backend. Missing
chat_message templates are a hard error at router build time. Router-
facing factories (Embedder/Scorer/Reranker/TokenCounter) re-resolve
ModelConfig per call so a model installed post-startup doesn't bind a
stub Backend="" config and silently fall into the loader's auto-
iterate path.

New 'vector_store' backend trace recorded inside localVectorStore on
every Search/Insert — including the backend-load-failure path that
previously vanished into an xlog.Warn — with outcome tagging
(hit/miss/empty_store/backend_load_error/find_error/insert_error/ok).
Companion cleanup drops misleading similarity:0 and input_tokens_count:0
from non-hit and text-mode traces.

Gallery local-store-development aliases to 'local-store' so the master
image satisfies pkg/model.LocalStoreBackend lookups from the embedding
cache.

Misc: llama-cpp TokenizeString reads the correct 'prompt' JSON key
(the original bug); ModelTokenize nil-guard; non-fatal mitm proxy
startup; PII 'route_local' renamed to 'allow' with docs/UI in sync;
model-editor footer no longer eats the edit area on small screens;
several config-editor template/dropdown/section fixes.

Tests: e2e router specs (casual/code-hint + long-conversation trim),
vector_store trace specs, lazy-factory specs, gallery dev-alias
resolution, Playwright trace badge + scroll regression.

Assisted-by: Claude:claude-opus-4-7 [Claude Code]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

* feat(backend): auto-size batch to context for embedding and rerank models

Embedding and rerank models pool over the whole input in a single physical batch (n_ubatch). With batch left at the 512 default, the backend rejects longer inputs with "input is too large to process", silently capping a large-context embedder (e.g. 8k/32k) at 512 tokens. Size n_batch to the context for these single-pass usecases, mirroring the existing FLAG_SCORE behaviour; an explicit batch: still wins.

Extracts EffectiveContextSize/EffectiveBatchSize from grpcModelOpts so the effective decode window has one home for other callers to reuse.

Adds an e2e-aio regression test that embeds a >512-token input. The AIO embedding model is switched to nomic-embed-text-v1.5 (2048 context) because the previous granite model was capped at 512 tokens and could not exercise the larger batch.

Assisted-by: claude-code:claude-opus-4-8 [Claude Code]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

* fix(gallery): raise arch-router scoring output cap via parallel:64

Scoring decodes the whole prompt+candidate in a single llama_decode and
reads one logit row per candidate token. The vendored llama.cpp server
caps causal output rows at n_parallel, so the default of 1 aborts with
GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max) on multi-token route
labels. Set options: [parallel:64] on both arch-router quant entries to
lift the cap; kv_unified (the grpc-server default) keeps the full context
per sequence, so this does not split the KV cache.

Assisted-by: claude-code:claude-opus-4-8 [Claude Code]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

---------

Signed-off-by: Richard Palethorpe <io@richiejp.com>

core/application/mitm.go

@@ -11,6 +11,29 @@ import (
 	"github.com/mudler/xlog"
 )
 
+// startMITMIfConfigured brings up the cloudproxy MITM listener when an
+// address is configured, treating any startup failure as non-fatal.
+//
+// The listener is opt-in middleware whose address is persisted in runtime
+// settings (/api/settings → runtime_settings.json) and replayed on every
+// boot. A bad value — e.g. a host the process can't bind, like a LAN IP
+// inside a container — must NOT abort the whole server: doing so crash-loops
+// with no way out, because the Settings UI used to correct the address can't
+// load if startup never completes. So on failure we log loudly and carry on;
+// the admin fixes the address via /api/settings, which calls RestartMITM.
+func startMITMIfConfigured(app *Application, options *config.ApplicationConfig) {
+	if options.MITMListen == "" {
+		return
+	}
+	if err := startMITMProxy(app, options); err != nil {
+		xlog.Error("mitm: cloudproxy listener failed to start — continuing without it",
+			"listen", options.MITMListen,
+			"error", err,
+			"hint", "fix the address via Settings (e.g. \":8082\" to bind all interfaces) and the listener will restart",
+		)
+	}
+}
+
 func startMITMProxy(app *Application, options *config.ApplicationConfig) error {
 	app.mitmMutex.Lock()
 	defer app.mitmMutex.Unlock()

core/application/mitm_test.go

@@ -0,0 +1,58 @@
+package application
+
+import (
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// minimal Application wired enough for startMITMProxy: an empty model
+// config loader (no host claims), CA written under a temp DataPath.
+func newMITMTestApp(dataPath string) (*Application, *config.ApplicationConfig) {
+	state, err := system.GetSystemState()
+	Expect(err).NotTo(HaveOccurred())
+	state.Model.ModelsPath = dataPath
+	opts := config.NewApplicationConfig(
+		config.WithSystemState(state),
+		config.WithDataPath(dataPath),
+	)
+	return newApplication(opts), opts
+}
+
+var _ = Describe("startMITMIfConfigured", func() {
+	It("does nothing when no listen address is configured", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		opts.MITMListen = ""
+
+		Expect(func() { startMITMIfConfigured(app, opts) }).NotTo(Panic())
+		Expect(app.mitmServer.Load()).To(BeNil(), "no listener should be stored when disabled")
+	})
+
+	// Regression: a persisted-but-unbindable MITM address (e.g. a LAN host
+	// inside a container) must not abort startup. startMITMIfConfigured
+	// swallows the bind error so the rest of LocalAI still comes up and the
+	// admin can fix the address via the Settings UI.
+	It("logs and continues when the listen address cannot be bound", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		// 192.0.2.1 is TEST-NET-1 (RFC 5737): guaranteed not assigned to any
+		// local interface, so bind fails deterministically without DNS.
+		opts.MITMListen = "192.0.2.1:8082"
+
+		Expect(func() { startMITMIfConfigured(app, opts) }).NotTo(Panic())
+		Expect(app.mitmServer.Load()).To(BeNil(), "failed listener must not be stored")
+	})
+
+	It("starts and stores the listener on a bindable address", func() {
+		app, opts := newMITMTestApp(GinkgoT().TempDir())
+		opts.MITMListen = "127.0.0.1:0" // OS-assigned free port
+
+		startMITMIfConfigured(app, opts)
+
+		srv := app.mitmServer.Load()
+		Expect(srv).NotTo(BeNil(), "listener should be stored on success")
+		DeferCleanup(srv.Stop)
+		Expect(srv.Addr()).NotTo(BeEmpty())
+	})
+})

core/application/router_factories.go

@@ -1,63 +1,120 @@
 package application
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
 )
 
-// adapterConfig resolves a model name to its runtime ModelConfig, or
-// nil when the name is unknown. Shared by the router-facing factories
-// below and by ModelConfigLookup.
+// adapterConfig resolves a model name to its runtime ModelConfig, or nil when
+// unknown. LoadModelConfigFileByNameDefaultOptions never returns nil — for an
+// unknown name it returns a defaults-filled stub with an empty Name (the YAML
+// `name:` field is required by Validate), which is how we tell the two apart.
 func (a *Application) adapterConfig(modelName string) *config.ModelConfig {
 	cfg, err := a.backendLoader.LoadModelConfigFileByNameDefaultOptions(modelName, a.applicationConfig)
-	if err != nil || cfg == nil {
+	if err != nil || cfg == nil || cfg.Name == "" {
 		return nil
 	}
 	return cfg
 }
 
-// ModelConfigLookup is the lookup function the router middleware's
-// classifier validator uses to confirm classifier_model declares
-// FLAG_SCORE before binding it.
+// ModelConfigLookup is the lookup the router middleware's classifier validator
+// uses to confirm classifier_model declares FLAG_SCORE before binding it.
 func (a *Application) ModelConfigLookup() func(modelName string) *config.ModelConfig {
 	return a.adapterConfig
 }
 
-// Scorer returns a backend.Scorer bound to the named model, or nil
-// when the model is unknown. Used as a method value (app.Scorer) by
-// router.ClassifierDeps — no factory-of-factory wrapper needed.
+// The router-facing factories below (Scorer, Embedder, Reranker, TokenCounter)
+// bind a model NAME at construction and re-resolve the CONFIG on every call.
+// Capturing the config at construction would bake in whatever state
+// adapterConfig saw first — including a stub returned before the YAML reached
+// bcl.configs (e.g. /import-model or gallery install racing startup). The
+// classifier registry caches factories by router-config fingerprint, so a
+// once-stale capture stays stale until the router config is edited.
+
 func (a *Application) Scorer(modelName string) backend.Scorer {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewScorer(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyScorer{app: a, modelName: modelName}
+}
+
+type lazyScorer struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyScorer) Score(ctx context.Context, prompt string, candidates []string) ([]backend.CandidateScore, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("scorer: model %q no longer available", l.modelName)
+	}
+	return backend.NewScorer(l.app.modelLoader, *cfg, l.app.applicationConfig).Score(ctx, prompt, candidates)
+}
+
+// TokenCounter returns a func so the middleware's literal field type accepts
+// it as a method value without importing core/http/middleware from here.
+func (a *Application) TokenCounter(modelName string) func(string) (int, error) {
+	if a.adapterConfig(modelName) == nil {
+		return nil
+	}
+	return func(text string) (int, error) {
+		cfg := a.adapterConfig(modelName)
+		if cfg == nil {
+			return 0, fmt.Errorf("token counter: model %q no longer available", modelName)
+		}
+		resp, err := backend.ModelTokenize(text, a.modelLoader, *cfg, a.applicationConfig)
+		if err != nil {
+			return 0, err
+		}
+		return len(resp.Tokens), nil
+	}
 }
 
-// Reranker returns a backend.Reranker bound to the named model, or
-// nil when unknown. The reranker model's `type:` (e.g. "colbert")
-// selects the scoring head inside the rerankers backend.
 func (a *Application) Reranker(modelName string) backend.Reranker {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewReranker(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyReranker{app: a, modelName: modelName}
+}
+
+type lazyReranker struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyReranker) Rerank(ctx context.Context, query string, documents []string) ([]backend.RerankResult, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("reranker: model %q no longer available", l.modelName)
+	}
+	return backend.NewReranker(l.app.modelLoader, *cfg, l.app.applicationConfig).Rerank(ctx, query, documents)
 }
 
-// Embedder returns a backend.Embedder bound to the named model, or
-// nil when unknown. Used by the router's L2 embedding cache.
 func (a *Application) Embedder(modelName string) backend.Embedder {
-	cfg := a.adapterConfig(modelName)
-	if cfg == nil {
+	if a.adapterConfig(modelName) == nil {
 		return nil
 	}
-	return backend.NewEmbedder(a.modelLoader, *cfg, a.applicationConfig)
+	return &lazyEmbedder{app: a, modelName: modelName}
 }
 
-// VectorStore returns a backend.VectorStore for the named collection,
-// or nil when the name is empty. Each router model gets its own
-// backend process via the model loader's cache keyed by storeName.
+type lazyEmbedder struct {
+	app       *Application
+	modelName string
+}
+
+func (l *lazyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
+	cfg := l.app.adapterConfig(l.modelName)
+	if cfg == nil {
+		return nil, fmt.Errorf("embedder: model %q no longer available", l.modelName)
+	}
+	return backend.NewEmbedder(l.app.modelLoader, *cfg, l.app.applicationConfig).Embed(ctx, text)
+}
+
+// VectorStore takes a store name, not a model name — no adapterConfig, no
+// staleness to avoid.
 func (a *Application) VectorStore(storeName string) backend.VectorStore {
 	return backend.NewVectorStore(a.modelLoader, a.applicationConfig, storeName)
 }

core/application/router_factories_test.go

@@ -0,0 +1,155 @@
+package application
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Regression: the router-facing factories used to capture
+// *config.ModelConfig at construction. A gallery install that raced
+// startup left a stub (Backend="") bound for the lifetime of the
+// classifier registry's cache entry, bypassing the user's `backend:`
+// config. These specs pin the lazy re-resolve.
+var _ = Describe("router_factories lazy config resolution", func() {
+	var (
+		tmpDir string
+		app    *Application
+	)
+
+	BeforeEach(func() {
+		var err error
+		tmpDir, err = os.MkdirTemp("", "router-factories-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		appCfg := &config.ApplicationConfig{
+			Context:     context.Background(),
+			SystemState: &system.SystemState{Model: system.Model{ModelsPath: tmpDir}},
+		}
+		app = &Application{
+			backendLoader:     config.NewModelConfigLoader(tmpDir),
+			modelLoader:       model.NewModelLoader(appCfg.SystemState),
+			applicationConfig: appCfg,
+		}
+	})
+
+	AfterEach(func() {
+		_ = os.RemoveAll(tmpDir)
+	})
+
+	// writeCfg seeds both the on-disk YAML and the in-memory cache —
+	// removing only the cache would fall through to file-read.
+	writeCfg := func(name, backend string) {
+		yaml := "name: " + name + "\nbackend: " + backend + "\nparameters:\n  model: " + name + ".bin\n"
+		Expect(os.WriteFile(filepath.Join(tmpDir, name+".yaml"), []byte(yaml), 0644)).To(Succeed())
+		Expect(app.backendLoader.LoadModelConfigsFromPath(tmpDir)).To(Succeed())
+		cfg, ok := app.backendLoader.GetModelConfig(name)
+		Expect(ok).To(BeTrue(), "config must be loaded before the spec runs")
+		Expect(cfg.Backend).To(Equal(backend))
+	}
+
+	// removeCfg purges both the cache and the YAML so LoadModelConfigFileByName
+	// returns the empty-stub case and adapterConfig returns nil.
+	removeCfg := func(name string) {
+		app.backendLoader.RemoveModelConfig(name)
+		Expect(os.Remove(filepath.Join(tmpDir, name+".yaml"))).To(Succeed())
+	}
+
+	Context("Embedder", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Embedder("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Embed call", func() {
+			writeCfg("emb-test", "llama-cpp")
+			emb := app.Embedder("emb-test")
+			Expect(emb).NotTo(BeNil())
+
+			// The factory must hold the NAME, not a captured config —
+			// otherwise stale captures survive cache invalidation.
+			lazy, ok := emb.(*lazyEmbedder)
+			Expect(ok).To(BeTrue(), "Embedder must return *lazyEmbedder")
+			Expect(lazy.modelName).To(Equal("emb-test"))
+
+			// Mutate the cached config. A lazy implementation sees the
+			// update on the next adapterConfig call; a captured-at-
+			// construction implementation would still see "llama-cpp".
+			app.backendLoader.UpdateModelConfig("emb-test", func(c *config.ModelConfig) {
+				c.Backend = "rerankers"
+			})
+			Expect(lazy.app.adapterConfig("emb-test").Backend).To(Equal("rerankers"))
+
+			// Remove the config entirely → Embed must surface the disappearance.
+			removeCfg("emb-test")
+			_, err := emb.Embed(context.Background(), "anything")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("Scorer", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Scorer("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Score call", func() {
+			writeCfg("score-test", "llama-cpp")
+			sc := app.Scorer("score-test")
+			Expect(sc).NotTo(BeNil())
+
+			lazy, ok := sc.(*lazyScorer)
+			Expect(ok).To(BeTrue(), "Scorer must return *lazyScorer")
+			Expect(lazy.modelName).To(Equal("score-test"))
+
+			removeCfg("score-test")
+			_, err := sc.Score(context.Background(), "prompt", []string{"a"})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("Reranker", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.Reranker("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each Rerank call", func() {
+			writeCfg("rerank-test", "rerankers")
+			rr := app.Reranker("rerank-test")
+			Expect(rr).NotTo(BeNil())
+
+			lazy, ok := rr.(*lazyReranker)
+			Expect(ok).To(BeTrue(), "Reranker must return *lazyReranker")
+			Expect(lazy.modelName).To(Equal("rerank-test"))
+
+			removeCfg("rerank-test")
+			_, err := rr.Rerank(context.Background(), "q", []string{"d"})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+
+	Context("TokenCounter", func() {
+		It("returns nil at construction for an unknown model", func() {
+			Expect(app.TokenCounter("missing")).To(BeNil())
+		})
+
+		It("re-resolves the model config on each call", func() {
+			writeCfg("tok-test", "llama-cpp")
+			tc := app.TokenCounter("tok-test")
+			Expect(tc).NotTo(BeNil())
+
+			removeCfg("tok-test")
+			_, err := tc("anything")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no longer available"))
+		})
+	})
+})

core/application/startup.go

@@ -462,11 +462,7 @@ func New(opts ...config.AppOption) (*Application, error) {
 	// traffic doesn't need a parallel config for MITM traffic.
 	// Runs after loadRuntimeSettingsFromFile so a listener configured
 	// via /api/settings is brought back up across restarts.
-	if options.MITMListen != "" {
-		if err := startMITMProxy(application, options); err != nil {
-			return nil, fmt.Errorf("mitm: startup: %w", err)
-		}
-	}
+	startMITMIfConfigured(application, options)
 
 	application.ModelLoader().SetBackendLoggingEnabled(options.EnableBackendLogging)