gallery: add Qwen3.5 9B Defiant Fable variants (#11335)

Add the MTP and plain Q4_K_M GGUF builds with their shared vision projector so LocalAI users can select accelerated or fallback llama.cpp inference.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=238ab6a9e321c17de8e120559f57efeedaeb1345
+AUDIO_CPP_VERSION?=7efbb58def443722ea540d931dd3debee3e4d5e8
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=6747e7718dd08f00b680d0c16231f2d59ec3747e
+# Upstream pin lives below as DS4_VERSION?=b0309611041655f4e45671cfd9c9886aff161406
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=6747e7718dd08f00b680d0c16231f2d59ec3747e
+DS4_VERSION?=b0309611041655f4e45671cfd9c9886aff161406
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=6b55d2c7504f482e7c8ec6cbf22a19f3778c522b
+IK_LLAMA_VERSION?=cf1aa57e1a0fabfd015831718fc99d1aec01ada5
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=ec730908a418b6032f9e69ded6186d3f042a7747
+CRISPASR_VERSION?=21901d3f7c23554f072964828363e49ddbc2dc68
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/crispasr/gocrispasr.go

@@ -67,7 +67,16 @@ const defaultTTSSampleRate = 24000
 // resampling, so the WAV header must match it. Returns ok=false for non-piper
 // models (key absent) or an unreadable file, letting the caller fall back to
 // defaultTTSSampleRate.
-func piperSampleRate(modelPath string) (int, bool) {
+func piperSampleRate(modelPath string) (rate int, ok bool) {
+	// A malformed metadata length can make gguf-parser-go panic before it can
+	// return an error. Keep a bad voice file from crash-looping the backend.
+	defer func() {
+		if recover() != nil {
+			rate = 0
+			ok = false
+		}
+	}()
+
 	// Only scalar architecture keys are read, so skip the large array metadata
 	// (phoneme map) and mmap the header - same rationale as pkg/vram's reader.
 	f, err := gguf.ParseGGUFFile(modelPath, gguf.UseMMap(), gguf.SkipLargeMetadata())
@@ -78,7 +87,7 @@ func piperSampleRate(modelPath string) (int, bool) {
 	if !ok || kv.ValueType != gguf.GGUFMetadataValueTypeUint32 {
 		return 0, false
 	}
-	rate := int(kv.ValueUint32())
+	rate = int(kv.ValueUint32())
 	if rate <= 0 {
 		return 0, false
 	}

backend/go/crispasr/gocrispasr_samplerate_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/binary"
+	"math"
 	"os"
 	"path/filepath"
 
@@ -102,6 +103,24 @@ var _ = Describe("piper sample rate", func() {
 			_, ok := piperSampleRate(p)
 			Expect(ok).To(BeFalse())
 		})
+
+		It("returns ok=false instead of panicking on a malformed string length", func() {
+			p := filepath.Join(GinkgoT().TempDir(), "malformed.gguf")
+			var b bytes.Buffer
+			b.WriteString("GGUF")
+			Expect(binary.Write(&b, binary.LittleEndian, uint32(3))).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(0))).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(1))).To(Succeed())
+			key := "general.name"
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(len(key)))).To(Succeed())
+			b.WriteString(key)
+			Expect(binary.Write(&b, binary.LittleEndian, ggufTypeString)).To(Succeed())
+			Expect(binary.Write(&b, binary.LittleEndian, uint64(math.MaxInt64))).To(Succeed())
+			Expect(os.WriteFile(p, b.Bytes(), 0o644)).To(Succeed())
+
+			_, ok := piperSampleRate(p)
+			Expect(ok).To(BeFalse())
+		})
 	})
 
 	// End-to-end through the built .so. Gated on CRISPASR_PIPER_MODEL_PATH (a

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=ea7f0c87cfe4c673263b4c201c596c7f1cbe2528
+STABLEDIFFUSION_GGML_VERSION?=c6beeef35526c6dc94b74a7fb69f9d2e6a2a7a12
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/vllm-cpp/Makefile

@@ -133,7 +133,26 @@ MLX_STAMP=
 MLX_CMAKE_ARGS=
 endif
 
+# govllmcpp.go mirrors vllm.h by hand, and the only guard against the two
+# drifting apart is the vllm_abi_version check inside registerLib - which fires
+# at runtime, on the user's machine, taking down every model load (issue
+# #11379). Compare the two here instead, so moving VLLM_CPP_VERSION past the
+# mirrors turns the build red while the header is still around to diff.
+abi-check: sources/vllm.cpp
+	@engine=$$(sed -n 's/^#define VLLM_ABI_VERSION \([0-9][0-9]*\).*/\1/p' sources/vllm.cpp/include/vllm.h); \
+	backend=$$(sed -n 's/^const abiVersion = \([0-9][0-9]*\).*/\1/p' govllmcpp.go); \
+	if [ -z "$$engine" ] || [ -z "$$backend" ]; then \
+		echo "vllm-cpp: cannot read the ABI version (engine='$$engine' backend='$$backend')" >&2; exit 1; \
+	fi; \
+	if [ "$$engine" != "$$backend" ]; then \
+		echo "vllm-cpp: ABI mismatch: vllm.cpp $(VLLM_CPP_VERSION) is v$$engine, govllmcpp.go mirrors v$$backend." >&2; \
+		echo "  Update the struct mirrors and abiVersion in govllmcpp.go (and the offsets in vllmcpp_test.go) to v$$engine." >&2; \
+		exit 1; \
+	fi; \
+	echo "vllm-cpp: ABI v$$engine matches the pinned engine"
+
 $(LIB): sources/vllm.cpp $(MLX_STAMP)
+	$(MAKE) abi-check
 	mkdir -p build && \
 	cd build && \
 	cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \
@@ -154,6 +173,8 @@ clean: purge
 purge:
 	rm -rf build
 
+.PHONY: abi-check
+
 .NOTPARALLEL:
 
 # The unit specs are pure Go (struct mirrors, option mapping, load

backend/go/vllm-cpp/README.md

@@ -6,7 +6,7 @@ safetensors + GGUF loading, CUDA / CPU / Metal / Vulkan) with no Python at
 inference time.
 
 The backend dlopens the engine's stable C ABI (`libvllm`, `include/vllm.h`,
-ABI v2) through purego:
+ABI v10) through purego:
 
 - `Load` -> `vllm_engine_load`: accepts a `.gguf` file or a HF-style model
   directory (`config.json` + safetensors). `context_size` maps to
@@ -29,6 +29,12 @@ ABI v2) through purego:
   LocalAI's Go-side grammar-constrained tool calling; JSON-schema / regex /
   choice constraints are also exposed by the ABI.
 
+The struct mirrors in `govllmcpp.go` are hand-written against one ABI version,
+and the engine refuses to load against any other. Moving `VLLM_CPP_VERSION` in
+the Makefile therefore means updating `abiVersion` plus the mirrors (and their
+offsets in `vllmcpp_test.go`) in the same change; `make abi-check` compares the
+pinned header against the bindings and the library build runs it first.
+
 Model config example:
 
 ```yaml

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -17,15 +17,21 @@ import (
 	"github.com/ebitengine/purego"
 )
 
-// abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 5
+// abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h). It must track
+// the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
+// the two against each other, because a mismatch is only caught at runtime by
+// registerLib, where it takes the backend down on every load (issue #11379).
+const abiVersion = 10
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params.
+// cModelParams mirrors vllm_model_params. The fields the backend does not set
+// are still mirrored: the engine reads the whole struct, so the Go value must
+// be the same size as the C one. Every one of them is inert when zeroed, which
+// is what keeps the engine byte-identical to the pre-v6 behavior.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
 	TokenizerConfigPath uintptr // const char*
@@ -35,11 +41,18 @@ type cModelParams struct {
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
+	SpeculativeConfig   uintptr // const char*; NULL = no speculation (ABI v6)
+	EnablePrefixCaching int32   // 0 = model default, 1 = on, 2 = off (ABI v7)
+	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
+	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
+	KVTransferConfig    uintptr // const char*; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // 0 = env-resolved (off), 1 = on, 2 = off (ABI v10)
+	_                   [4]byte
 }
 
-// cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
-// included). Padding matches the C compiler's: the uint64 seed is 8-aligned,
-// and each pointer following an int32 is 8-aligned.
+// cSamplingParams mirrors vllm_sampling_params (structured fields included).
+// Padding matches the C compiler's: the uint64 seed is 8-aligned, and each
+// pointer following an int32 is 8-aligned.
 type cSamplingParams struct {
 	Temperature          float32
 	TopP                 float32
@@ -65,6 +78,10 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
+	// Per-request custom logits processor (ABI v8). Left NULL: a Go callback
+	// would have to run inside the sampler's decode step for every token.
+	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
+	LogitsProcessorUserData uintptr // void*, passed back to the callback
 }
 
 // cCompletion mirrors vllm_completion.

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,10 +16,17 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v2)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v10)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
+	It("declares the ABI version the pinned engine reports", func() {
+		// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
+		// Moving the pin past this without growing the mirrors below ships a
+		// backend that refuses every load at startup (issue #11379).
+		Expect(abiVersion).To(Equal(10))
+	})
+
 	It("cModelParams matches vllm_model_params", func() {
 		var p cModelParams
 		Expect(unsafe.Offsetof(p.ModelPath)).To(Equal(uintptr(0)))
@@ -30,10 +37,16 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumSeqs)).To(Equal(uintptr(28)))
 		Expect(unsafe.Offsetof(p.ToolParser)).To(Equal(uintptr(32)))
 		Expect(unsafe.Offsetof(p.ReasoningParser)).To(Equal(uintptr(40)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.SpeculativeConfig)).To(Equal(uintptr(48)))
+		Expect(unsafe.Offsetof(p.EnablePrefixCaching)).To(Equal(uintptr(56)))
+		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
+		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
+		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
+		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
 	})
 
-	It("cSamplingParams matches vllm_sampling_params (ABI v2)", func() {
+	It("cSamplingParams matches vllm_sampling_params", func() {
 		var p cSamplingParams
 		Expect(unsafe.Offsetof(p.Temperature)).To(Equal(uintptr(0)))
 		Expect(unsafe.Offsetof(p.TopP)).To(Equal(uintptr(4)))
@@ -55,7 +68,9 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.NStructuredChoice)).To(Equal(uintptr(96)))
 		Expect(unsafe.Offsetof(p.StructuredGrammar)).To(Equal(uintptr(104)))
 		Expect(unsafe.Offsetof(p.StructuredJSONObject)).To(Equal(uintptr(112)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessor)).To(Equal(uintptr(120)))
+		Expect(unsafe.Offsetof(p.LogitsProcessorUserData)).To(Equal(uintptr(128)))
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
 	})
 
 	It("cCompletion matches vllm_completion", func() {

core/gallery/estimate_warm.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/concurrency"
 	"github.com/mudler/LocalAI/pkg/system"
 	"github.com/mudler/LocalAI/pkg/vram"
 	"github.com/mudler/xlog"
@@ -101,7 +102,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 		return
 	}
 
-	go func() {
+	concurrency.SafeGo(func() {
 		started := time.Now()
 
 		models, err := AvailableGalleryModelsCached(galleries, systemState)
@@ -131,7 +132,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 
 		for i := 0; i < cfg.Concurrency; i++ {
 			wg.Add(1)
-			go func() {
+			concurrency.SafeGo(func() {
 				defer wg.Done()
 				for m := range cursor {
 					// Per entry, not for the run: one unreachable weight file
@@ -164,7 +165,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 
 					cancel()
 				}
-			}()
+			})
 		}
 
 	feed:
@@ -183,7 +184,7 @@ func WarmEstimateCache(ctx context.Context, galleries []config.Gallery, systemSt
 			return
 		}
 		xlog.Info("gallery caches warmed", "estimates", warmed, "variants", warmedVariants, "of", len(models), "took", time.Since(started).Round(time.Second))
-	}()
+	})
 }
 
 // EstimateWarmConfigFromEnv reads the warm-up bounds from the environment,

core/gallery/estimate_warm_test.go

@@ -1,11 +1,20 @@
 package gallery_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
+	"math"
+	"net/http"
+	"net/http/httptest"
 	"os"
+	"path/filepath"
+	"time"
 
+	gguf "github.com/gpustack/gguf-parser-go"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
@@ -57,6 +66,46 @@ var _ = Describe("VRAM estimate warm-up", func() {
 		Consistently(func() bool { return true }, "100ms").Should(BeTrue())
 	})
 
+	It("does not crash the server when remote GGUF metadata is malformed", func() {
+		payload := warmMalformedGGUF()
+		requested := make(chan struct{})
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			select {
+			case <-requested:
+			default:
+				close(requested)
+			}
+			http.ServeContent(w, r, "model.gguf", time.Time{}, bytes.NewReader(payload))
+		}))
+		DeferCleanup(server.Close)
+
+		galleryPath := filepath.Join(state.Model.ModelsPath, "malformed-gallery.yaml")
+		index, err := yaml.Marshal([]gallery.GalleryModel{{Metadata: gallery.Metadata{
+			Name: "malformed-gguf",
+			AdditionalFiles: []gallery.File{{
+				Filename: "model.gguf",
+				URI:      server.URL + "/model.gguf",
+			}},
+		}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, index, 0600)).To(Succeed())
+
+		cfg := gallery.DefaultEstimateWarmConfig
+		cfg.Limit = 1
+		cfg.Concurrency = 1
+		cfg.Contexts = []uint32{8192}
+		gallery.WarmEstimateCache(context.Background(), []config.Gallery{{
+			Name: "malformed",
+			URL:  "file://" + galleryPath,
+		}}, state, cfg)
+
+		Eventually(requested, "2s").Should(BeClosed())
+		// The warm-up is detached. Give its parser time to consume the response;
+		// before the recovery boundary, that goroutine panicked and killed the
+		// entire test process (and the LocalAI server in production).
+		Consistently(func() bool { return true }, "300ms").Should(BeTrue())
+	})
+
 	Describe("configuration from the environment", func() {
 		AfterEach(func() {
 			os.Unsetenv("LOCALAI_VRAM_WARM_LIMIT")
@@ -113,3 +162,19 @@ var _ = Describe("VRAM estimate warm-up", func() {
 	})
 
 })
+
+func warmMalformedGGUF() []byte {
+	payload := make([]byte, 0, 128)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMagicGGUFLe))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFVersionV3))
+	payload = binary.LittleEndian.AppendUint64(payload, 0)
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	key := "tokenizer.ggml.tokens"
+	payload = binary.LittleEndian.AppendUint64(payload, uint64(len(key)))
+	payload = append(payload, key...)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeArray))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	payload = binary.LittleEndian.AppendUint64(payload, math.MaxUint64)
+	return payload
+}

core/gallery/importers/llama-cpp.go

@@ -401,7 +401,10 @@ func maybeApplyMTPDefaults(modelConfig *config.ModelConfig, details Details, cfg
 		}
 	}()
 
-	f, err := gguf.ParseGGUFFileRemote(ctx, probeURL)
+	// MTP markers are architecture scalars. Avoid allocating tokenizer and
+	// other large arrays from an untrusted remote header; panic recovery cannot
+	// contain a fatal out-of-memory condition.
+	f, err := gguf.ParseGGUFFileRemote(ctx, probeURL, gguf.SkipLargeMetadata())
 	if err != nil {
 		xlog.Debug("[mtp-importer] failed to read remote GGUF header for MTP detection", "uri", probeURL, "error", err)
 		return

core/http/react-ui/e2e/traces-live.spec.js

@@ -20,3 +20,46 @@ test('marks an API trace with no response status as in progress', async ({ page
   await expect(row.locator('[title="In progress"]')).toBeVisible()
   await expect(row.locator('.fa-check-circle')).toHaveCount(0)
 })
+
+// Regression for #11376: switching from Backend Traces back to API Traces
+// used to crash the page. `traces` holds whichever list was fetched last, so
+// right after `setActiveTab('api')` — before the refetch effect lands — the
+// API table renders the previous tab's backend rows, which carry no
+// `response` envelope. The status column must tolerate that instead of
+// dereferencing `trace.response.status` and tearing down the React tree.
+test('switching from backend to API traces with a response-less row does not crash', async ({ page }) => {
+  const pageErrors = []
+  page.on('pageerror', (e) => pageErrors.push(e.message))
+
+  await page.route('**/api/traces?*', route => route.fulfill({
+    json: [{
+      id: 'api-1',
+      timestamp: '2026-08-05T02:00:00Z',
+      request: { method: 'POST', path: '/v1/chat/completions' },
+      response: { status: 200 },
+    }],
+    headers: { 'X-Total-Count': '1' },
+  }))
+  await page.route('**/api/backend-traces?*', route => route.fulfill({
+    json: [{
+      id: 'backend-1',
+      type: 'llm',
+      timestamp: '2026-08-05T02:00:00Z',
+      model_name: 'mock-model',
+      summary: 'generated a reply',
+    }],
+    headers: { 'X-Total-Count': '1' },
+  }))
+
+  await page.goto('/app/traces')
+  await expect(page.locator('tbody tr').filter({ hasText: '/v1/chat/completions' })).toBeVisible()
+
+  await page.getByRole('button', { name: /Backend Traces/ }).click()
+  await expect(page.locator('tbody tr').filter({ hasText: 'generated a reply' })).toBeVisible()
+
+  await page.getByRole('button', { name: /API Traces/ }).click()
+  // The stale backend row renders in the API table for one frame; the status
+  // column falls back to a neutral placeholder rather than throwing.
+  await expect(page.locator('tbody tr').filter({ hasText: '/v1/chat/completions' })).toBeVisible()
+  expect(pageErrors).toEqual([])
+})

core/http/react-ui/src/pages/Traces.jsx

@@ -667,6 +667,8 @@ export default function Traces() {
                     <td>
                       {trace.response?.status === 0
                         ? <span className="badge badge-info">Running</span>
+                        : trace.response?.status == null
+                        ? <span className="badge badge--soft">-</span>
                         : <span className={`badge ${trace.response.status < 400 ? 'badge-success' : 'badge-error'}`}>{trace.response.status}</span>}
                     </td>
                     <td><LatencyCell ns={trace.duration} max={slowestTrace} /></td>

docker-compose.distributed.yaml

@@ -74,6 +74,9 @@ services:
       GODEBUG: "netdns=go"
       # Paths
       MODELS_PATH: /models
+      # Avoid probing remote gallery GGUF metadata during container startup.
+      # Remove this line or set a positive limit to opt back into cache warming.
+      LOCALAI_VRAM_WARM_LIMIT: "0"
     volumes:
       - frontend_models:/models
       - frontend_data:/data

docker-compose.yaml

@@ -18,6 +18,9 @@ services:
       - .env
     environment:
       - MODELS_PATH=/models
+      # Avoid probing remote gallery GGUF metadata during container startup.
+      # Remove this line or set a positive limit to opt back into cache warming.
+      - LOCALAI_VRAM_WARM_LIMIT=0
     #  - DEBUG=true
     ## Agents (LocalAGI) - https://localai.io/features/agents/
     #  - LOCALAI_DISABLE_AGENTS=false

docs/content/advanced/vram-management.md

@@ -477,6 +477,11 @@ then on.
 | `LOCALAI_VRAM_WARM_LIMIT` | `300` | How many gallery entries to warm at startup, estimates and variants alike. Set to `0` to disable the warm-up entirely. |
 | `LOCALAI_VRAM_WARM_CONCURRENCY` | `4` | How many estimates to run at once. |
 
+The provided Docker Compose configurations set `LOCALAI_VRAM_WARM_LIMIT=0`
+as a defensive default, so container startup does not probe remote GGUF files.
+Remove that override or set it to a positive number to opt into background
+warming.
+
 ```bash
 # Air-gapped, or you would rather not make the requests at all
 LOCALAI_VRAM_WARM_LIMIT=0 local-ai run

docs/data/version.json

@@ -1,3 +1,3 @@
 {
-  "version": "v4.7.1"
+  "version": "v4.8.0"
 }

gallery/index.yaml

@@ -1,4 +1,101 @@
 ---
+- &qwen3-5-9b-defiant-fable
+  name: "qwen3.5-9b-defiant-fable-mtp"
+  variants:
+    - model: qwen3.5-9b-defiant-fable
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF
+  description: |
+    Qwen3.5 9B Defiant Fable is an Apache-2.0 multimodal fine-tune for
+    reasoning, coding, creative writing, and roleplay. It retains the 256K
+    context window and vision support of Qwen3.5 while reducing refusals.
+    This default entry uses the NEO-imatrix Q4_K_M build with multi-token
+    prediction enabled for faster generation.
+  license: apache-2.0
+  icon: https://huggingface.co/DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/resolve/main/defiant-fable-9b.png
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen3.5
+    - reasoning
+    - coding
+    - creative-writing
+    - uncensored
+    - vision
+    - multimodal
+    - mtp
+  last_checked: "2026-08-04"
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - vision
+    mmproj: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+    options:
+      - use_jinja:true
+      - spec_type:draft-mtp
+      - spec_n_max:6
+      - spec_p_min:0.75
+    parameters:
+      model: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q4_K_M.gguf
+      sha256: d7eb4fac9389d53fa576f64a6ff53e914a00bc7705dc354d1065887565147320
+    - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf
+      sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383
+- !!merge <<: *qwen3-5-9b-defiant-fable
+  name: "qwen3.5-9b-defiant-fable"
+  variants: []
+  description: |
+    Qwen3.5 9B Defiant Fable in the plain NEO-imatrix Q4_K_M GGUF format.
+    This fallback offers the same multimodal reasoning, coding, and creative
+    capabilities without enabling multi-token prediction.
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen3.5
+    - reasoning
+    - coding
+    - creative-writing
+    - uncensored
+    - vision
+    - multimodal
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+      - vision
+    mmproj: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.5-9b-defiant-fable/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-Q4_K_M.gguf
+      sha256: d33db5e583b9c9251402e876443791bc979f12af934bfb0630eadfb456279f84
+    - filename: llama-cpp/mmproj/qwen3.5-9b-defiant-fable/mmproj-BF16.gguf
+      uri: huggingface://DavidAU/Qwen3.5-9B-The-Defiant-Fable-Uncensored-Heretic-NEO-IMATRIX-MAX-MTP-GGUF/mmproj-BF16.gguf
+      sha256: 853698ce7aa6c7ba732478bad280240969ddf7b0fcbf93900046f63903a83383
 - &nemotron-3-embed-1b
   name: "nemotron-3-embed-1b-q4"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
@@ -1992,7 +2089,7 @@
   files:
     - filename: ds4flash.gguf
       uri: https://huggingface.co/unsloth/DeepSeek-V4-Flash-GGUF
-      sha256: ba1d64ad8d77038124839956b614db2e889daa1a4ddc83060bb06ccb5a1d7461
+      sha256: ea3dc48cb9797ea1bfaa8a74d8a819756b06b16e8fbaa30728ad2cd0a643c605
 - name: "qwopus3.6-35b-a3b-coder-mtp"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -2807,6 +2904,86 @@
     - filename: llama-cpp/mmproj/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/mmproj-F32.gguf
       sha256: 32f7ea0600c07272547da401d460f8abbd980f3a57b69d6df87be0e2505e0b9c
       uri: https://huggingface.co/Jackrong/Qwopus3.6-27B-Coder-Compat-MTP-GGUF/resolve/main/mmproj-F32.gguf
+- &qwen3-5-9b-hauhaucs-aggressive
+  name: "qwen3.5-9b-hauhaucs-aggressive"
+  variants:
+    - model: qwen3.5-9b-hauhaucs-aggressive-q8
+  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+  urls:
+    - https://huggingface.co/Qwen/Qwen3.5-9B
+    - https://huggingface.co/HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive
+  description: |
+    Qwen3.5 9B Aggressive is HauhauCS's refusal-removed fine-tune of the
+    multimodal Qwen3.5 9B model. It retains the base model's reasoning, tool
+    use, image and video understanding, and 262K-token native context window.
+
+    This entry uses the balanced Q4_K_M GGUF quantization and includes the
+    matching BF16 multimodal projector. The Q8_0 variant offers higher fidelity.
+  license: "apache-2.0"
+  tags:
+    - llm
+    - gguf
+    - cpu
+    - gpu
+    - qwen
+    - multimodal
+    - uncensored
+  icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg
+  last_checked: "2026-08-04"
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+      sha256: 2ca636d9e81d3d23ca9b60c234fe185d30ec082eeba69ce770fdb0c76559a4f5
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+    - filename: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q4_K_M/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+      sha256: 05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+- !!merge <<: *qwen3-5-9b-hauhaucs-aggressive
+  name: "qwen3.5-9b-hauhaucs-aggressive-q8"
+  variants: []
+  description: |
+    Qwen3.5 9B Aggressive is HauhauCS's refusal-removed fine-tune of the
+    multimodal Qwen3.5 9B model. It retains the base model's reasoning, tool
+    use, image and video understanding, and 262K-token native context window.
+
+    This entry uses the higher-fidelity Q8_0 GGUF quantization and includes the
+    matching BF16 multimodal projector.
+  overrides:
+    backend: llama-cpp
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    mmproj: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+    options:
+      - use_jinja:true
+    parameters:
+      model: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+      sha256: 99e7f2201c0046b05d2825e4d8be6a2efad2b87b071cd55d37bdd9fbe201a58b
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf
+    - filename: llama-cpp/mmproj/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-Q8_0/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+      sha256: 05f662501f8bd45607b079723a3e238a4e888fd085a10a53f4057a0e250f6934
+      uri: huggingface://HauhauCS/Qwen3.5-9B-Uncensored-HauhauCS-Aggressive/mmproj-Qwen3.5-9B-Uncensored-HauhauCS-Aggressive-BF16.gguf
 # DFlash speculative-decoding pairs (upstream llama.cpp `draft-dflash`).
 # Each entry ships a full target model plus a small block-diffusion drafter
 # (z-lab DFlash, converted with upstream convert_hf_to_gguf.py, GGUF arch

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gofrs/flock v0.13.0
 	github.com/google/go-containerregistry v0.21.6
 	github.com/google/uuid v1.6.0
-	github.com/gpustack/gguf-parser-go v0.24.0
+	github.com/gpustack/gguf-parser-go v0.25.0
 	github.com/hpcloud/tail v1.0.0
 	github.com/ipfs/go-log v1.0.5
 	github.com/jaypipes/ghw v0.24.0

go.sum

@@ -666,8 +666,8 @@ github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
-github.com/gpustack/gguf-parser-go v0.24.0 h1:tdJceXYp9e5RhE9RwVYIuUpir72Jz2D68NEtDXkKCKc=
-github.com/gpustack/gguf-parser-go v0.24.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
+github.com/gpustack/gguf-parser-go v0.25.0 h1:1AMBhMKtI24nTtn588Bq53FqNiOvEw1x9Nb4HbRrThs=
+github.com/gpustack/gguf-parser-go v0.25.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=

pkg/vram/gguf_reader.go

@@ -2,6 +2,7 @@ package vram
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	gguf "github.com/gpustack/gguf-parser-go"
@@ -10,7 +11,18 @@ import (
 
 type defaultGGUFReader struct{}
 
-func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFMeta, error) {
+func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (meta *GGUFMeta, err error) {
+	// gguf-parser-go parses lengths supplied by the file and has historically
+	// panicked on values that cannot fit in a Go slice. Metadata can come from
+	// an untrusted remote host, and this reader is also used by a background
+	// gallery worker, where an escaped panic would terminate the whole server.
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			meta = nil
+			err = fmt.Errorf("read GGUF metadata: parser panic: %v", recovered)
+		}
+	}()
+
 	u := downloader.URI(uri)
 	urlStr := u.ResolveURL()
 
@@ -28,7 +40,10 @@ func (defaultGGUFReader) ReadMetadata(ctx context.Context, uri string) (*GGUFMet
 	if !u.LooksLikeHTTPURL() {
 		return nil, nil
 	}
-	f, err := gguf.ParseGGUFFileRemote(ctx, urlStr)
+	// The estimator only consumes architecture scalars. Tokenizer arrays can
+	// be very large and are unnecessary here, so avoid downloading or
+	// allocating them for remote files just as the local path does above.
+	f, err := gguf.ParseGGUFFileRemote(ctx, urlStr, gguf.SkipLargeMetadata())
 	if err != nil {
 		return nil, err
 	}

pkg/vram/gguf_reader_test.go

@@ -0,0 +1,115 @@
+package vram_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/vram"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("DefaultGGUFReader", func() {
+	It("reads architecture scalars from a valid remote GGUF", func() {
+		server := serveGGUF(validRemoteGGUF())
+
+		meta, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(meta).To(Equal(&vram.GGUFMeta{
+			BlockCount:           32,
+			EmbeddingLength:      4096,
+			HeadCount:            32,
+			HeadCountKV:          8,
+			MaximumContextLength: 8192,
+		}))
+	})
+
+	It("rejects an overflowing tokenizer array without allocating it", func() {
+		server := serveGGUF(malformedGGUFArray(math.MaxUint64))
+
+		_, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).NotTo(ContainSubstring("parser panic"),
+			"large tokenizer metadata should be skipped with a bounds error")
+	})
+
+	It("converts a parser panic from malformed string metadata to an error", func() {
+		server := serveGGUF(malformedGGUFString(uint64(math.MaxInt64)))
+
+		_, err := vram.DefaultGGUFReader().ReadMetadata(context.Background(), server.URL+"/model.gguf")
+
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("parser panic"))
+	})
+})
+
+func serveGGUF(payload []byte) *httptest.Server {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeContent(w, r, "model.gguf", time.Time{}, bytes.NewReader(payload))
+	}))
+	DeferCleanup(server.Close)
+	return server
+}
+
+func malformedGGUFString(length uint64) []byte {
+	payload := ggufHeader(1)
+	payload = appendGGUFString(payload, "general.name")
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, length)
+	return payload
+}
+
+func validRemoteGGUF() []byte {
+	payload := ggufHeader(6)
+	payload = appendGGUFStringValue(payload, "general.architecture", "llama")
+	payload = appendGGUFUint32(payload, "llama.block_count", 32)
+	payload = appendGGUFUint32(payload, "llama.embedding_length", 4096)
+	payload = appendGGUFUint32(payload, "llama.attention.head_count", 32)
+	payload = appendGGUFUint32(payload, "llama.attention.head_count_kv", 8)
+	payload = appendGGUFUint32(payload, "llama.context_length", 8192)
+	return payload
+}
+
+func malformedGGUFArray(itemLength uint64) []byte {
+	payload := ggufHeader(1)
+	payload = appendGGUFString(payload, "tokenizer.ggml.tokens")
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeArray))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	payload = binary.LittleEndian.AppendUint64(payload, 1)
+	payload = binary.LittleEndian.AppendUint64(payload, itemLength)
+	return payload
+}
+
+func ggufHeader(metadataCount uint64) []byte {
+	payload := make([]byte, 0, 128)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMagicGGUFLe))
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFVersionV3))
+	payload = binary.LittleEndian.AppendUint64(payload, 0)
+	payload = binary.LittleEndian.AppendUint64(payload, metadataCount)
+	return payload
+}
+
+func appendGGUFString(payload []byte, value string) []byte {
+	payload = binary.LittleEndian.AppendUint64(payload, uint64(len(value)))
+	return append(payload, value...)
+}
+
+func appendGGUFStringValue(payload []byte, key, value string) []byte {
+	payload = appendGGUFString(payload, key)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeString))
+	return appendGGUFString(payload, value)
+}
+
+func appendGGUFUint32(payload []byte, key string, value uint32) []byte {
+	payload = appendGGUFString(payload, key)
+	payload = binary.LittleEndian.AppendUint32(payload, uint32(gguf.GGUFMetadataValueTypeUint32))
+	return binary.LittleEndian.AppendUint32(payload, value)
+}

website/content/blog/what-landed-in-localai-4-8.md

@@ -3,12 +3,12 @@ title: "What landed in LocalAI 4.8"
 date: 2026-08-04
 author: "Ettore Di Giacinto"
 category: "Release"
-tags: ["release", "vllm.cpp", "audio.cpp", "3d", "gallery", "distributed", "performance"]
-summary: "A new inference engine, 3D generation, one backend that serves six audio endpoints, and a web interface 3.48x lighter. 374 pull requests in twenty-one days."
+tags: ["release", "vllm.cpp", "audio.cpp", "3d", "agent", "gallery", "distributed", "performance"]
+summary: "A new inference engine, a terminal agent in the CLI, 3D generation, and a web interface 3.48x lighter. 386 pull requests in twenty-two days."
 extracss: ["blog.css"]
 ---
 
-LocalAI 4.8.0 is out, after twenty-one days and 374 merged pull requests. There are three new things LocalAI can do, and a lot of repair work on things it already did.
+LocalAI 4.8.0 is out, after twenty-two days and 386 merged pull requests. There are four new things LocalAI can do, and a lot of repair work on things it already did.
 
 The full notes list everything. This post covers the parts that change what you do day to day, with the pull request numbers so you can read the diffs.
 
@@ -144,6 +144,20 @@ The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS
 <figcaption>trellis2-4b, 2,502,928 vertices and 5,012,118 triangles, turning in the browser. The remesh slider below it is the print path.</figcaption>
 </figure>
 
+## `local-ai chat` stopped being a REPL
+
+`local-ai chat` used to be a chat prompt in a terminal. It is now an agent, and it is the [nib](https://github.com/mudler/nib) harness compiled straight into the binary: tool use behind an approval gate, sub-agents, MCP servers, plugins and skills, auto-configured against your own instance. Nothing extra to install.
+
+```bash
+local-ai chat                      # the agent, pointed at your models
+echo "what is 2+2" | local-ai chat --cli
+local-ai chat --init zsh           # Ctrl+Space from any shell prompt
+```
+
+That last one prints a shell integration script (zsh, bash or fish), so you can pull the agent up from wherever you already are instead of opening something else.
+
+It runs shell commands now, so every tool call goes through an approval prompt you control, and read-only ones like `ls` and `cat` run without asking. If you had habits around the old REPL, a few things moved: `/clear` is gone and `/compact` is the closest thing, `/models` and `/model <name>` mean what they always meant, and switching model keeps the conversation instead of starting over ([#11291](https://github.com/mudler/LocalAI/pull/11291)).
+
 ## One backend, six audio endpoints
 
 The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine. One backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.

website/layouts/index.html

@@ -19,7 +19,7 @@
           <div><b class="tnum" data-count="{{ .Site.Data.stats.stars }}">0</b><span>GitHub stars</span></div>
           <div><b class="tnum" data-count="73">0</b><span>Backends</span></div>
           <div><b class="tnum" data-count="{{ len .Site.Data.engines.engines }}">0</b><span>Engines we wrote</span></div>
-          <div><b class="tnum" data-count="1585">0</b><span>Models, one click</span></div>
+          <div><b class="tnum" data-count="1255">0</b><span>Models, one click</span></div>
         </div>
       </div>
       <div class="fd">
@@ -58,7 +58,7 @@
       </div>
       <div class="duo__m rv">
         <figure class="screen" style="margin:0">
-          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,585 models</b></figcaption>
+          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,255 models</b></figcaption>
           <video src="/media/gallery.mp4" muted loop playsinline preload="none" data-lazy aria-label="Installing a model from the LocalAI gallery"></video>
         </figure>
       </div>
@@ -328,7 +328,7 @@
   <div class="shell">
     <div class="bars rv" aria-hidden="true"><i></i><i></i><i></i><i></i></div>
     <p class="kicker rv">The gallery</p>
-    <h2 class="rv mt1" style="max-width:20ch">1,585 models. No notebook, no conversion script.</h2>
+    <h2 class="rv mt1" style="max-width:20ch">1,255 models. No notebook, no conversion script.</h2>
     <div class="cards">
       <a class="cd rv" href="/docs/getting-started/models/"><p class="cd__k">Quantizations</p><h3>201 APEX builds</h3>
         <p>Every tier of every model we quantize, ranked against the hardware you actually have and installed with one click.</p><span class="cd__go">Browse the gallery →</span></a>