From c541dbeef40786ac004d48bb993b79f13a945ea7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 20:44:43 +0000 Subject: [PATCH 01/75] fix(distributed): check a node answers before scheduling onto it A node's status comes from its HTTP heartbeat. Backend installs travel over NATS. The two are independent, so a worker that dies stops answering on the bus at once but stays healthy in the database until its heartbeat ages out. Inside that window the scheduler picked a node it could not reach, and the request failed with "no responders available" rather than moving to a node that was up. The scheduler now probes the node it selected and, when nothing answers, marks it unhealthy and selects again. The demotion is what makes the retry terminate: the next selection reads only healthy nodes. It also tells the other frontends what this one learned, so the cluster does not rediscover a dead worker one failed request at a time. Only nats.ErrNoResponders counts as absent. A worker that answers slowly stays eligible, because dropping it would cost capacity that is really there. The probe reuses the models.running subject: a new subject would go unanswered by workers that have not been upgraded, and every one of them would then look dead. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/interfaces.go | 1 + core/services/nodes/model_router_test.go | 4 + core/services/nodes/router.go | 56 +++++---- core/services/nodes/router_liveness.go | 60 +++++++++ .../nodes/router_nats_liveness_test.go | 118 ++++++++++++++++++ core/services/nodes/router_test.go | 31 ++++- core/services/nodes/unloader.go | 25 ++++ docs/content/features/distributed-mode.md | 6 + 8 files changed, 277 insertions(+), 24 deletions(-) create mode 100644 core/services/nodes/router_liveness.go create mode 100644 core/services/nodes/router_nats_liveness_test.go diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index c204752de..93399bc14 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -60,6 +60,7 @@ type ModelRouter interface { GetNodeLabels(ctx context.Context, nodeID string) ([]NodeLabel, error) FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) LoadedReplicaStats(ctx context.Context, modelName string, candidateNodeIDs []string) ([]ReplicaCandidate, error) + MarkUnhealthy(ctx context.Context, nodeID string) error LoadJobStore } diff --git a/core/services/nodes/model_router_test.go b/core/services/nodes/model_router_test.go index 43002006a..13e36b12d 100644 --- a/core/services/nodes/model_router_test.go +++ b/core/services/nodes/model_router_test.go @@ -250,3 +250,7 @@ var _ = Describe("ModelRouterAdapter", func() { }) }) }) + +func (f *fakeModelRouterForSmartRouter) MarkUnhealthy(_ context.Context, _ string) error { + return nil +} diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 465b2d44e..14b19092e 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1089,34 +1089,44 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // If freeSlotNodes is empty (everyone full), candidateNodeIDs is whatever // it was — we'll fall through to eviction below. - var node *BackendNode - - if estimatedVRAM > 0 { - if candidateNodeIDs != nil { - node, err = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) - } else { - node, err = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) - } - if err != nil { - xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", - "required_vram", vram.FormatBytes(estimatedVRAM), "error", err) - } - } - - if node == nil { - if candidateNodeIDs != nil { - node, err = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) - if err != nil { - node, err = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) + // Node choice is wrapped in a liveness check: a node's stored status comes + // from its HTTP heartbeat, which is a different channel from the bus that + // carries the install. A worker that has died stops answering on the bus at + // once but stays healthy in the database until its heartbeat ages out, so + // without this the scheduler could commit to a node it cannot reach. + selectNode := func() *BackendNode { + var candidate *BackendNode + var selErr error + if estimatedVRAM > 0 { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindNodeWithVRAMFromSet(ctx, estimatedVRAM, candidateNodeIDs) + } else { + candidate, selErr = r.registry.FindNodeWithVRAM(ctx, estimatedVRAM) } - } else { - node, err = r.registry.FindIdleNode(ctx) - if err != nil { - node, err = r.registry.FindLeastLoadedNode(ctx) + if selErr != nil { + xlog.Warn("No nodes with enough VRAM, falling back to standard scheduling", + "required_vram", vram.FormatBytes(estimatedVRAM), "error", selErr) } } + + if candidate == nil { + if candidateNodeIDs != nil { + candidate, selErr = r.registry.FindIdleNodeFromSet(ctx, candidateNodeIDs) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNodeFromSet(ctx, candidateNodeIDs) + } + } else { + candidate, selErr = r.registry.FindIdleNode(ctx) + if selErr != nil { + candidate, _ = r.registry.FindLeastLoadedNode(ctx) + } + } + } + return candidate } + node := r.pickReachableNode(ctx, selectNode) + // 4. Preemptive eviction: if no suitable node found, evict the LRU model with zero in-flight if node == nil { evictedNode, evictErr := r.evictLRUAndFreeNode(ctx) diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go new file mode 100644 index 000000000..88646162f --- /dev/null +++ b/core/services/nodes/router_liveness.go @@ -0,0 +1,60 @@ +package nodes + +import ( + "context" + "errors" + + "github.com/mudler/xlog" + "github.com/nats-io/nats.go" +) + +// maxNodeLivenessRetries bounds how many unreachable nodes a single scheduling +// attempt discards before giving up. Each discarded node is marked unhealthy, +// so the bound only has to cover one burst of dead workers rather than the +// whole fleet. +const maxNodeLivenessRetries = 3 + +// nodeAnswersOnBus reports whether a node still has a live subscription. +// +// Only nats.ErrNoResponders means "absent". Any other outcome, a timeout or a +// transport hiccup, leaves the node eligible: wrongly excluding a node that is +// merely slow costs real capacity, while the install that follows already +// reports its own failure. When no command sender is configured there is no bus +// to consult and every node is treated as reachable, which preserves the +// behaviour of deployments that do not run one. +func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { + if r.unloader == nil || node == nil { + return true + } + err := r.unloader.PingNode(node.ID) + return !errors.Is(err, nats.ErrNoResponders) +} + +// pickReachableNode calls selectNode until it yields a node that still answers +// on the bus, and returns nil when it cannot find one. +// +// A node that does not answer is marked unhealthy before the next attempt. That +// both removes it from the next selection, which queries only healthy nodes, +// and tells every other scheduler in the cluster what this one just learned, so +// the discovery is not repeated one failed request at a time. +func (r *SmartRouter) pickReachableNode(ctx context.Context, selectNode func() *BackendNode) *BackendNode { + for range maxNodeLivenessRetries { + node := selectNode() + if node == nil { + return nil + } + if r.nodeAnswersOnBus(node) { + return node + } + xlog.Warn("Scheduled node is not answering on the bus, marking unhealthy and re-scheduling", + "node", node.Name, "nodeID", node.ID) + if err := r.registry.MarkUnhealthy(ctx, node.ID); err != nil { + // Without the demotion the next selection would hand back the same + // node, so stop rather than spin. + xlog.Warn("Failed to mark unreachable node unhealthy", + "node", node.Name, "nodeID", node.ID, "error", err) + return nil + } + } + return nil +} diff --git a/core/services/nodes/router_nats_liveness_test.go b/core/services/nodes/router_nats_liveness_test.go new file mode 100644 index 000000000..ec4820c9f --- /dev/null +++ b/core/services/nodes/router_nats_liveness_test.go @@ -0,0 +1,118 @@ +package nodes + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A node's stored status comes from its HTTP heartbeat, but work is dispatched +// over NATS. A worker that dies stops answering on the bus immediately and +// keeps its healthy status until the heartbeat ages out, so the scheduler could +// commit to a node it could not reach. The request then failed outright with +// "no responders available" rather than moving to a node that was actually up. +var _ = Describe("Scheduling past a node that left the bus", func() { + var ( + reg *fakeModelRouter + fake *fakeUnloader + router *SmartRouter + ) + + newNode := func(id string) *BackendNode { + return &BackendNode{ID: id, Name: id, Address: id + ":50051"} + } + + // selectorReturning hands back each node in turn, mimicking a scheduler + // that re-picks after the previous choice was demoted. + selectorReturning := func(nodes ...*BackendNode) func() *BackendNode { + i := 0 + return func() *BackendNode { + if i >= len(nodes) { + return nil + } + n := nodes[i] + i++ + return n + } + } + + BeforeEach(func() { + reg = &fakeModelRouter{} + fake = &fakeUnloader{deadNodes: map[string]bool{}} + router = NewSmartRouter(reg, SmartRouterOptions{Unloader: fake}) + }) + + It("passes over a node that no longer answers and takes one that does", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("alive-node")) + Expect(fake.pingCalls).To(Equal([]string{"dead-node", "alive-node"})) + }) + + It("demotes the absent node so other schedulers stop choosing it", func() { + dead, alive := newNode("dead-node"), newNode("alive-node") + fake.deadNodes["dead-node"] = true + + router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) + + Expect(reg.markedUnhealthy).To(Equal([]string{"dead-node"})) + }) + + It("takes the first node when it answers, without probing further", func() { + first, second := newNode("first"), newNode("second") + + picked := router.pickReachableNode(context.Background(), selectorReturning(first, second)) + + Expect(picked.ID).To(Equal("first")) + Expect(fake.pingCalls).To(Equal([]string{"first"})) + }) + + It("gives up rather than spinning when every node is gone", func() { + a, b, c, d := newNode("a"), newNode("b"), newNode("c"), newNode("d") + for _, id := range []string{"a", "b", "c", "d"} { + fake.deadNodes[id] = true + } + + picked := router.pickReachableNode(context.Background(), selectorReturning(a, b, c, d)) + + Expect(picked).To(BeNil()) + Expect(len(fake.pingCalls)).To(BeNumerically("<=", maxNodeLivenessRetries)) + }) + + It("stops when the demotion itself fails, so it cannot loop on one node", func() { + dead := newNode("dead-node") + fake.deadNodes["dead-node"] = true + reg.markUnhealthyErr = errors.New("database is down") + + picked := router.pickReachableNode(context.Background(), selectorReturning(dead, dead, dead)) + + Expect(picked).To(BeNil()) + Expect(fake.pingCalls).To(Equal([]string{"dead-node"})) + }) + + // Only a no-responders answer proves absence. Excluding a node that is + // merely slow would cost real capacity. + It("keeps a node that answers slowly or errors for another reason", func() { + slow := newNode("slow-node") + fake.pingErr = errors.New("timeout waiting for reply") + + picked := router.pickReachableNode(context.Background(), selectorReturning(slow)) + + Expect(picked).ToNot(BeNil()) + Expect(picked.ID).To(Equal("slow-node")) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("treats every node as reachable when no command sender is configured", func() { + plain := NewSmartRouter(reg, SmartRouterOptions{}) + node := newNode("only-node") + + Expect(plain.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) + }) +}) diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 96db9b93f..0f0938335 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -17,6 +17,7 @@ import ( "github.com/mudler/LocalAI/pkg/distributedhdr" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/nats-io/nats.go" ggrpc "google.golang.org/grpc" "google.golang.org/protobuf/proto" "gorm.io/gorm" @@ -61,6 +62,10 @@ func (f *fakeFileStager) ListRemoteDir(_ context.Context, _, _ string) ([]string // fakeModelRouter implements ModelRouter with configurable return values. type fakeModelRouter struct { + // markedUnhealthy records nodes demoted by the scheduler's liveness check. + markedUnhealthy []string + markUnhealthyErr error + fakeLoadJobStore // FindAndLockNodeWithModel returns @@ -474,7 +479,15 @@ type fakeUnloader struct { stopCalls []string // "nodeID:model" stopErr error unloadCalls []string - unloadErr error + + // deadNodes names the nodes PingNode reports as absent from the bus, and + // pingCalls records every node it was asked about, in order. + deadNodes map[string]bool + pingCalls []string + // pingErr is returned for nodes not in deadNodes, so a spec can model a + // node that is reachable but answering badly. + pingErr error + unloadErr error } // installCall captures the args we care about when asserting that the @@ -532,6 +545,22 @@ func (f *fakeUnloader) UnloadModelOnNode(nodeID, modelName string) error { return f.unloadErr } +func (f *fakeModelRouter) MarkUnhealthy(_ context.Context, nodeID string) error { + f.markedUnhealthy = append(f.markedUnhealthy, nodeID) + return f.markUnhealthyErr +} + +func (f *fakeUnloader) PingNode(nodeID string) error { + f.mu.Lock() + f.pingCalls = append(f.pingCalls, nodeID) + dead := f.deadNodes[nodeID] + f.mu.Unlock() + if dead { + return nats.ErrNoResponders + } + return f.pingErr +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 8d47d71a6..3b1cd15b8 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -36,6 +36,10 @@ type NodeCommandSender interface { ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error UnloadModelOnNode(nodeID, modelName string) error + // PingNode reports whether the node is still subscribed on the bus. It + // returns nats.ErrNoResponders when nothing answers for the node, which is + // the only condition callers may read as "this node cannot be given work". + PingNode(nodeID string) error } // RemoteUnloaderAdapter implements NodeCommandSender and model.RemoteModelUnloader @@ -360,6 +364,27 @@ func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendL return messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply](a.nats, subject, messaging.BackendListRequest{}, 30*time.Second) } +// PingNode checks that a worker still has a live subscription on the bus. +// +// A node's status in the database comes from its HTTP heartbeat, which is a +// separate channel from NATS. A worker that has died stops answering on NATS +// at once but keeps its healthy status until the heartbeat ages out, so the +// scheduler could pick a node that could not be given work and the request +// failed with "no responders available". +// +// It reuses the models.running subject rather than a dedicated ping subject on +// purpose: a new subject would go unanswered by any worker that has not been +// upgraded yet, and this check would then report every one of them as dead. +// The worker answers out of its in-memory process table, so a live node +// replies immediately, and NATS reports no-responders without waiting out the +// timeout. +func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { + subject := messaging.SubjectNodeModelsRunning(nodeID) + _, err := messaging.RequestJSON[messaging.ModelsRunningRequest, messaging.ModelsRunningReply]( + a.nats, subject, messaging.ModelsRunningRequest{}, 5*time.Second) + return err +} + // ListRunningModels asks a worker node which model backend processes it // currently has running, via NATS request-reply. // diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 54c3abbab..667408fbb 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,6 +1020,12 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A request fails with `nats: no responders available for request`:** +- The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. +- The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. +- Only a no-responders answer counts as absent. A worker that answers slowly stays eligible, because excluding it would cost capacity that is really there. +- Check the worker process is running and its NATS connection is up. `Scheduled node is not answering on the bus` in the frontend log names each node demoted this way. + **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file to the worker under `/../staging/ephemeral/`. The worker deletes these 6 hours after the request that needed them, and sweeps every 30 minutes plus once at startup, so a worker that crashed mid-request still reclaims the space. - Releases before this sweep existed kept every staged input for the lifetime of the worker. Delete `/../staging/ephemeral/` on an affected worker once, as the user the worker runs as; the sweep keeps it bounded from then on. From e6269e3cdd09f902bf22671bf9593eeaf3bbb30e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 21:07:29 +0000 Subject: [PATCH 02/75] fix(distributed): reclaim replica slots held by abandoned loads A replica row in staging or loading holds its slot, because slot allocation counts every state except unloading. Nothing ever reclaimed such a row: every reconciler pass and the router's eviction query filter state = "loaded", and the per-model probe skips rows without an address, which is exactly what a row that never finished loading has. So a worker that dropped out mid-transfer left a row that pinned the only replica slot for that model on that node. Scheduling then found no free slot and eviction found nothing it was allowed to evict, and the request failed with "no replica slot on and eviction failed: all models busy". The state persisted until an operator intervened. The reconciler now reclaims a row stuck before serving when no load job is driving it. Ownership is decided by the job's LastProgress heartbeat, not by elapsed time: staging a large checkpoint legitimately runs for a long while without touching the replica row, so a deadline would either be a model-size cliff or reclaim a healthy transfer. That heartbeat is the same signal job takeover already trusts. Any error reading the job leaves the slot held, because holding one for another pass costs a scheduling opportunity while a wrong reclaim restarts a multi-gigabyte transfer. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/reconciler.go | 8 +- .../nodes/reconciler_abandoned_load.go | 98 +++++++++++++ .../nodes/reconciler_abandoned_load_test.go | 133 ++++++++++++++++++ docs/content/features/distributed-mode.md | 6 + 4 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 core/services/nodes/reconciler_abandoned_load.go create mode 100644 core/services/nodes/reconciler_abandoned_load_test.go diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index f7fefc835..88ca28bbe 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -278,8 +278,9 @@ func (rc *ReplicaReconciler) reconcileOnce(ctx context.Context) { // reconcileState runs the state-reconciliation passes: drain pending backend // ops for freshly-healthy nodes, reconcile registry rows against what workers -// report they are running, then port-probe whatever is left. All passes are -// best-effort: a failure on one node doesn't stop the rest. +// report they are running, port-probe whatever is left, then reclaim replica +// slots held by loads nobody is driving. All passes are best-effort: a failure +// on one node doesn't stop the rest. // // Order matters. The worker pass runs first and refreshes updated_at for every // model a worker vouches for, which takes those rows out of the port prober's @@ -292,6 +293,9 @@ func (rc *ReplicaReconciler) reconcileState(ctx context.Context) { rc.reconcileNodeProcesses(ctx) rc.probeLoadedModels(ctx) rc.sweepLeakedInFlight(ctx) + // Runs last: the passes above can move a row into a serving state, and a + // row that just became loaded is no longer this sweeper's business. + rc.reclaimAbandonedLoads(ctx) } // drainPendingBackendOps retries queued backend ops whose next_retry_at has diff --git a/core/services/nodes/reconciler_abandoned_load.go b/core/services/nodes/reconciler_abandoned_load.go new file mode 100644 index 000000000..9b6b7b654 --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load.go @@ -0,0 +1,98 @@ +package nodes + +import ( + "context" + "errors" + "time" + + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +const ( + // abandonedLoadGrace is how long a replica row may sit in a pre-serving + // state before the sweeper will consider it at all. + // + // It exists to cover the window between creating the replica row and + // writing the load job that vouches for it. Without it a load could be + // reclaimed in the moment before its own job row exists. It is not the + // thing that protects a long transfer: the job heartbeat does that. + abandonedLoadGrace = 5 * time.Minute +) + +// preServingStates are the replica states that hold a slot without being able +// to serve a request. NextFreeReplicaIndex counts every state except +// "unloading", so a row parked in one of these occupies capacity while +// answering nothing. +var preServingStates = []string{"loading", "staging"} + +// reclaimAbandonedLoads removes replica rows whose load will never finish. +// +// The other reconciler passes and the router's eviction query all filter +// state = "loaded", and the per-model probe skips rows without an address, so +// nothing reclaimed a row that never got that far. On a node with one replica +// slot per model, a single interrupted transfer made the model unschedulable +// there until an operator intervened: scheduling saw no free slot, and eviction +// found nothing it was allowed to evict. +// +// A row is abandoned when no live load job vouches for it. Ownership is decided +// by the job's LastProgress heartbeat rather than elapsed time, because staging +// a large checkpoint legitimately runs for a long while without touching the +// replica row. That is the same signal job takeover already trusts, so a +// transfer this sweeper reclaims is one no replica is still driving. +func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { + if rc.db == nil { + return + } + + cutoff := time.Now().Add(-abandonedLoadGrace) + var stuck []NodeModel + if err := rc.db.WithContext(ctx). + Where("state IN ? AND updated_at < ?", preServingStates, cutoff). + Find(&stuck).Error; err != nil { + xlog.Warn("Reconciler: failed to list replicas stuck before serving", "error", err) + return + } + + now := time.Now() + for _, row := range stuck { + if rc.loadStillRunning(ctx, row.ModelName, now) { + continue + } + if err := rc.registry.RemoveNodeModel(ctx, row.NodeID, row.ModelName, row.ReplicaIndex); err != nil { + xlog.Warn("Reconciler: failed to reclaim abandoned load", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, + "state", row.State, "error", err) + continue + } + xlog.Warn("Reconciler: reclaimed a replica slot held by a load nobody is driving", + "node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, "state", row.State) + } +} + +// loadStillRunning reports whether a load job is actively driving this model. +// +// A missing job means nobody is loading it. A failed job has already given up. +// An orphaned job stopped heartbeating, which is the condition another replica +// uses to take it over, so the transfer behind it is not progressing either. +// Any error reading the job is treated as "still running": leaving a slot held +// for one more pass costs a scheduling opportunity, while removing a row out +// from under a live transfer would restart a multi-gigabyte load. +func (rc *ReplicaReconciler) loadStillRunning(ctx context.Context, modelName string, now time.Time) bool { + job, err := rc.registry.GetLoadJob(ctx, modelName) + if errors.Is(err, gorm.ErrRecordNotFound) { + return false + } + if err != nil { + xlog.Warn("Reconciler: cannot read load job, leaving the replica slot held", + "model", modelName, "error", err) + return true + } + if job == nil { + return false + } + if job.State == LoadJobStateFailed { + return false + } + return !job.IsOrphaned(now) +} diff --git a/core/services/nodes/reconciler_abandoned_load_test.go b/core/services/nodes/reconciler_abandoned_load_test.go new file mode 100644 index 000000000..6740b13aa --- /dev/null +++ b/core/services/nodes/reconciler_abandoned_load_test.go @@ -0,0 +1,133 @@ +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/testutil" +) + +// A replica row in loading or staging holds its slot: NextFreeReplicaIndex +// counts every state except unloading. Nothing reclaimed such a row. Every +// reconciler sweep and the router's eviction query filter state = "loaded", and +// the per-model health probe skips rows with no address, which is exactly what a +// row that never finished loading has. So a worker that dropped out mid-transfer +// left a row that pinned the only replica slot on that node for that model, and +// the next request failed with "no replica slot ... all models busy". +// +// Elapsed time alone cannot decide this: staging a large checkpoint legitimately +// runs for tens of minutes. The load job's LastProgress heartbeat is the +// discriminator, the same signal job takeover already trusts. +var _ = Describe("ReplicaReconciler — abandoned load sweeper", func() { + var ( + db *gorm.DB + registry *NodeRegistry + node *BackendNode + rc *ReplicaReconciler + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + node = &BackendNode{Name: "n1", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"} + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + rc = NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, DB: db}) + }) + + // seedReplica creates a replica row in the given state, aged so it is past + // the sweeper's grace period unless stated otherwise. + seedReplica := func(model, state string, age time.Duration) { + Expect(db.Create(&NodeModel{ + ID: model + "-row", + NodeID: node.ID, + ModelName: model, + State: state, + UpdatedAt: time.Now().Add(-age), + }).Error).To(Succeed()) + } + + seedJob := func(model, state string, sinceProgress time.Duration) { + Expect(db.Create(&ModelLoadJob{ + TrackingKey: model, + State: state, + OwnerReplica: "someone", + LastProgress: time.Now().Add(-sinceProgress), + CreatedAt: time.Now().Add(-sinceProgress), + UpdatedAt: time.Now().Add(-sinceProgress), + }).Error).To(Succeed()) + } + + rowExists := func(model string) bool { + var count int64 + Expect(db.Model(&NodeModel{}).Where("model_name = ?", model).Count(&count).Error).To(Succeed()) + return count > 0 + } + + It("reclaims a staging row whose load job has stopped heartbeating", func() { + seedReplica("abandoned", "staging", time.Hour) + seedJob("abandoned", LoadJobStateStaging, 30*time.Minute) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("abandoned")).To(BeFalse()) + }) + + It("reclaims a loading row that has no load job at all", func() { + seedReplica("orphan", "loading", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("orphan")).To(BeFalse()) + }) + + It("keeps a long transfer whose job is still heartbeating", func() { + // The row itself is old, because staging does not touch it. Only the + // job proves the transfer is alive. + seedReplica("big-model", "staging", time.Hour) + seedJob("big-model", LoadJobStateStaging, time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("big-model")).To(BeTrue(), "a live transfer must never be reclaimed") + }) + + It("leaves a freshly created row alone while its job row is still being written", func() { + seedReplica("just-started", "loading", time.Second) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("just-started")).To(BeTrue()) + }) + + It("does not touch loaded replicas, which the other sweeps own", func() { + seedReplica("serving", "loaded", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("serving")).To(BeTrue()) + }) + + It("frees the slot so the model can be scheduled on that node again", func() { + seedReplica("wedged", "staging", time.Hour) + seedJob("wedged", LoadJobStateFailed, time.Minute) + + _, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).To(MatchError(ErrNoFreeSlot), "precondition: the stuck row holds the only slot") + + rc.reclaimAbandonedLoads(context.Background()) + + idx, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1) + Expect(err).ToNot(HaveOccurred()) + Expect(idx).To(Equal(0)) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 667408fbb..d82616024 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1020,6 +1020,12 @@ Notes: - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. +**A model cannot be scheduled on a node that looks free (`no replica slot ... all models busy, cannot evict`):** +- A replica row in `staging` or `loading` holds its slot: slot allocation counts every state except `unloading`. If a worker drops out mid-transfer, that row never reaches `loaded`, and eviction only ever considers `loaded` replicas, so on a node with one replica slot per model the model became unschedulable there. +- The reconciler now reclaims a replica row stuck before serving when no load job is still driving it, and the freed slot is immediately reusable. +- Liveness is decided by the load job's progress heartbeat, not by elapsed time. Staging a large checkpoint legitimately runs for a long time without touching the replica row, so a transfer that is still progressing is never reclaimed however long it takes. +- `Reconciler: reclaimed a replica slot held by a load nobody is driving` names each row reclaimed this way. + **A request fails with `nats: no responders available for request`:** - The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. - The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. From 7a78ec82ebcea79c360ea35b04a5a61348c1ecea Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:46:34 +0200 Subject: [PATCH 03/75] chore(model-gallery): :arrow_up: update checksum (#11690) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 31cb6a925..c9a240db2 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -50,8 +50,8 @@ use_tokenizer_template: true files: - filename: llama-cpp/models/Huihui-Qwen3.8-27B-abliterated-bf16/Huihui-Qwen3.8-27B-abliterated-bf16.gguf - sha256: a64a5e5464d7d0ea7ffcbc937cf28f8a7bc9b0a6e87be6034e6b854418d5abd5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/Huihui-Qwen3.8-27B-abliterated-bf16.gguf + sha256: b880f2042df16a9a493800c83f1ee16b7cd46e8ca695f193655940a1949e9097 - filename: llama-cpp/mmproj/Huihui-Qwen3.8-27B-abliterated-bf16/mmproj-model-bf16.gguf sha256: c9a09064683620bea3d3bfed5d4462e1a97a7d2fff7e5045d6862a0a85eeb5b5 uri: https://huggingface.co/huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/resolve/main/mmproj-model-bf16.gguf @@ -1180,7 +1180,7 @@ files: - filename: llama-cpp/models/nemotron-3.5-lightning-30b-a3b/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf uri: huggingface://ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf - sha256: d4cc4c9fffaa356db8b49cfaba9233609cfb29b4dd89d827665210dc1cc8dbbb + sha256: b0e25ce2d301930e706d549a59d18c5969dcec664fdf7466435287b03fefda36 - &muse-glimmer-30b name: "muse-glimmer-30b" variants: @@ -18663,11 +18663,11 @@ model: SmolVLM2-256M-Video-Instruct-Q8_0.gguf files: - filename: SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: af7ce9951a2f46c4f6e5def253e5b896ca5e417010e7a9949fdc9e5175c27767 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 1202d1c54493bddff5b0ecbc36fcb7520ff720b9fa3d7224aeb293581f90529a - filename: mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf - sha256: d34913a588464ff7215f086193e0426a4f045eaba74456ee5e2667d8ed6798b1 uri: huggingface://ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf + sha256: 05d5751132244a6ebd64cba9b34898c0d874b2cb78159d758e1d4da3aad91581 - name: qwen3-30b-a3b url: github:mudler/LocalAI/gallery/qwen3.yaml@master urls: From eadc005b861a715b2efa296bee60caaf11210740 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:49:32 +0200 Subject: [PATCH 04/75] chore: :arrow_up: Update antirez/ds4 to `c1d4597a80e300b803dc642519718f2c999589da` (#11685) :arrow_up: Update antirez/ds4 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ds4/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index bdf126122..5e73c1328 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +# Upstream pin lives below as DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +DS4_VERSION?=c1d4597a80e300b803dc642519718f2c999589da DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 3953448f600ebf0e5a4b305c281d3d5790316a07 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 23 Aug 2026 22:17:57 +0000 Subject: [PATCH 05/75] fix(distributed): resync stored config revisions at startup The controller pins a model's replicas to a stored revision and rejects any request carrying a different one. Nothing ever re-derived that value from the configuration on disk: it moved only on an edit, a gallery install, or a peer's change broadcast. An inference request may only establish a revision, never replace one. So any other way for the two to diverge left the model permanently unroutable. A configuration edited while a frontend was down lands there, and so does a change in what the revision is computed over: an upgrade that alters the hashed form leaves every stored revision describing a configuration that no longer exists. The only recovery was deleting the row by hand, which is not something a cluster should need. Each frontend now reconciles the stored revisions against the loaded configurations at startup and republishes the ones that disagree. Only those: republishing quarantines every replica loaded under the old revision, so doing it for a model that did not drift would unload a healthy replica for nothing. A model with no stored revision has never been served and is left for its first request to establish. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/application/startup.go | 10 ++ core/services/modeladmin/revision_resync.go | 118 +++++++++++++++ .../modeladmin/revision_resync_test.go | 142 ++++++++++++++++++ docs/content/features/distributed-mode.md | 5 +- 4 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 core/services/modeladmin/revision_resync.go create mode 100644 core/services/modeladmin/revision_resync_test.go diff --git a/core/application/startup.go b/core/application/startup.go index 66a813162..a5aec5090 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -373,6 +373,16 @@ func New(opts ...config.AppOption) (*Application, error) { cfgLoaderOpts := options.ToConfigLoaderOptions() modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup) gs.SetModelRevisionLifecycle(modelRevisionLifecycle) + // Bring the controller's stored revisions back in line with the + // configuration on disk. An inference request may only establish a + // revision, never replace one, so a model whose stored value had + // drifted stayed unroutable until someone deleted the row. + if err := modeladmin.ResyncModelConfigRevisions(options.Context, + application.ModelConfigLoader(), + modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle), + ); err != nil { + xlog.Warn("Failed to resync model config revisions", "error", err) + } gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { // ApplyRemoteChange honors the op: a "delete" prunes the element // (a reload-from-path is additive and cannot drop it), anything diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go new file mode 100644 index 000000000..4724f60e0 --- /dev/null +++ b/core/services/modeladmin/revision_resync.go @@ -0,0 +1,118 @@ +package modeladmin + +import ( + "context" + "errors" + "fmt" + + "github.com/mudler/xlog" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/config" +) + +// ErrNoStoredRevision reports that the controller holds no revision for a +// model, which is the normal state for one that has never been served. +var ErrNoStoredRevision = gorm.ErrRecordNotFound + +// RevisionStore is the controller state this resync reads and corrects. +type RevisionStore interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) + ApplyConfigRevisions(ctx context.Context, transitions []ModelRevisionTransition) (int, error) +} + +// RevisionReader is the read half, satisfied by the node registry. +type RevisionReader interface { + GetModelConfigRevision(ctx context.Context, modelName string) (string, error) +} + +type revisionStore struct { + RevisionReader + lifecycle ModelRevisionLifecycle +} + +func (s revisionStore) ApplyConfigRevisions(ctx context.Context, t []ModelRevisionTransition) (int, error) { + return s.lifecycle.ApplyConfigRevisions(ctx, t) +} + +// NewRevisionStore pairs the registry that holds the stored revisions with the +// lifecycle that publishes new ones. Returns nil when either half is missing, +// which ResyncModelConfigRevisions treats as "nothing to reconcile". +func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) RevisionStore { + if reader == nil || lifecycle == nil { + return nil + } + return revisionStore{RevisionReader: reader, lifecycle: lifecycle} +} + +// ResyncModelConfigRevisions makes the controller's stored revision for each +// model agree with what this build computes from the configuration on disk. +// +// The stored revision is what every inference request is checked against, but +// nothing ever re-derived it from the persisted configuration: it moved only on +// an edit, a gallery install, or a peer's change broadcast. Any other way for +// the two to diverge left the model permanently unroutable, because an +// inference request may only establish a revision, never replace one. A +// configuration edited while this frontend was down, or a change in what the +// revision is computed over, both landed there, and the only recovery was +// deleting the row by hand. +// +// Running this at startup makes that self-correcting. Only a model whose stored +// revision disagrees is republished, so replicas of models that did not drift +// keep serving: republishing is not free, it quarantines every replica loaded +// under the old revision. +// +// A model with no stored revision is left alone. It has never been served, and +// inventing controller state for it here would quarantine nothing and describe +// a model that may never be requested. +func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, store RevisionStore) error { + if loader == nil || store == nil { + return nil + } + + var transitions []ModelRevisionTransition + for _, cfg := range loader.GetAllModelsConfigs() { + want, err := config.ModelConfigRevision(&cfg) + if err != nil { + return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) + } + + stored, err := store.GetModelConfigRevision(ctx, cfg.Name) + if errors.Is(err, ErrNoStoredRevision) { + continue + } + if err != nil { + return fmt.Errorf("read stored config revision for %q: %w", cfg.Name, err) + } + if stored == want { + continue + } + + xlog.Warn("Stored model config revision disagrees with the configuration on disk, republishing", + "model", cfg.Name, "stored", shortRevision(stored), "computed", shortRevision(want)) + transitions = append(transitions, ModelRevisionTransition{ + ModelName: cfg.Name, ConfigRevision: want, Disabled: cfg.IsDisabled(), + }) + } + + if len(transitions) == 0 { + return nil + } + if _, err := store.ApplyConfigRevisions(ctx, transitions); err != nil { + return fmt.Errorf("republish model config revisions: %w", err) + } + xlog.Info("Republished model config revisions to match the configuration on disk", "models", len(transitions)) + return nil +} + +// shortRevision trims a revision for log output; the leading bytes identify it +// well enough to tell two apart. +func shortRevision(revision string) string { + if revision == "" { + return "(none)" + } + if len(revision) > 12 { + return revision[:12] + } + return revision +} diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go new file mode 100644 index 000000000..3979ccf75 --- /dev/null +++ b/core/services/modeladmin/revision_resync_test.go @@ -0,0 +1,142 @@ +package modeladmin + +import ( + "context" + "errors" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/system" +) + +// stubRevisionStore stands in for the controller's stored revisions. +type stubRevisionStore struct { + stored map[string]string + getErr error + applied []ModelRevisionTransition + applyEr error +} + +func (s *stubRevisionStore) GetModelConfigRevision(_ context.Context, name string) (string, error) { + if s.getErr != nil { + return "", s.getErr + } + rev, ok := s.stored[name] + if !ok { + return "", ErrNoStoredRevision + } + return rev, nil +} + +func (s *stubRevisionStore) ApplyConfigRevisions(_ context.Context, t []ModelRevisionTransition) (int, error) { + s.applied = append(s.applied, t...) + return 0, s.applyEr +} + +// The controller pins a model's replicas to a stored revision and rejects any +// request carrying a different one. Nothing ever re-derived that stored value +// from the configuration on disk: it only moved on an edit, a gallery install +// or a peer's change event. So whenever the stored value stopped matching what +// this build computes for an unchanged file, every request for that model was +// rejected until an operator deleted the row by hand. +var _ = Describe("ResyncModelConfigRevisions", func() { + var ( + dir string + loader *config.ModelConfigLoader + store *stubRevisionStore + appConfig *config.ApplicationConfig + ) + + write := func(name, body string) { + Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) + } + + revisionOf := func(name string) string { + cfg, ok := loader.GetModelConfig(name) + Expect(ok).To(BeTrue()) + rev, err := config.ModelConfigRevision(&cfg) + Expect(err).ToNot(HaveOccurred()) + return rev + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + appConfig = config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader = config.NewModelConfigLoader(dir) + store = &stubRevisionStore{stored: map[string]string{}} + }) + + load := func() { + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + } + + It("republishes the revision when the stored one no longer matches the config on disk", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\ncontext_size: 4096\n") + load() + store.stored["drifted"] = "a-revision-from-an-earlier-build" + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + Expect(store.applied[0].ConfigRevision).To(Equal(revisionOf("drifted"))) + }) + + It("leaves a model alone when the stored revision already matches", func() { + write("agreed", "name: agreed\nbackend: llama-cpp\n") + load() + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing") + }) + + // A model nobody has served has no stored revision. Creating one here would + // invent controller state for a model that may never be requested; the first + // request establishes it. + It("does not create state for a model that has never been served", func() { + write("never-served", "name: never-served\nbackend: llama-cpp\n") + load() + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(BeEmpty()) + }) + + It("republishes only the models that actually drifted", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + write("agreed", "name: agreed\nbackend: llama-cpp\ncontext_size: 2048\n") + load() + store.stored["drifted"] = "stale" + store.stored["agreed"] = revisionOf("agreed") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + + Expect(store.applied).To(HaveLen(1)) + Expect(store.applied[0].ModelName).To(Equal("drifted")) + }) + + It("reports a store failure instead of continuing silently", func() { + write("drifted", "name: drifted\nbackend: llama-cpp\n") + load() + store.stored["drifted"] = "stale" + store.applyEr = errors.New("database is down") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + }) + + It("skips a model whose stored revision cannot be read rather than guessing", func() { + write("unreadable", "name: unreadable\nbackend: llama-cpp\n") + load() + store.getErr = errors.New("connection reset") + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(store.applied).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index d82616024..70d63684c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1041,8 +1041,9 @@ Notes: **Requests fail with `stale model config revision` although nobody edited the model:** - A model's stored revision must describe its persisted configuration. Releases before this fix also hashed the per-request prediction parameters, so the first request after a restart pinned the revision to its own `temperature`, `top_p`, `stop` and similar values. Every later request that sent different values was then rejected. - Upgrade the frontend replicas first. After the upgrade the revision is stamped when the configuration is loaded, so it no longer depends on the request body. -- The stored revision does not heal on its own, because the recorded value belongs to no persisted configuration. Clear it once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` -- Saving any edit for the model through the API or the WebUI has the same effect, because an edit publishes the current revision. +- Each frontend now reconciles the stored revisions against the configuration on disk at startup, and republishes any that disagree, so a drifted revision heals on the next restart. Only models that actually drifted are republished, because republishing quarantines the replicas loaded under the old revision. +- A model that has never been served has no stored revision and is left alone; its first request establishes one. +- On a release without that reconciliation, clear the row once per affected model so the next request establishes the correct revision: `DELETE FROM model_config_states WHERE model_name = '';` Saving any edit through the API or the WebUI has the same effect. **Port conflicts on workers:** - Each model gets its own gRPC process on an incrementing port (50051, 50052, ...) From 5c9d8190d9b056935f135fac71b834816c95eea8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 06:40:30 +0000 Subject: [PATCH 06/75] fix(distributed): resync revisions after the configs are loaded The resync added in 3953448f6 ran before LoadModelConfigsFromPath, so it read an empty loader, reconciled nothing and reported success. The symptom was a stored revision that stayed stale across restarts while the log showed no complaint, which is exactly what the resync was meant to prevent. Move the call after the configs are loaded, and refuse to treat an empty loader as a clean run: reconciling zero models is indistinguishable from reconciling correctly, and that is what hid the mis-ordered call. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/application/startup.go | 29 ++++++++++++------- core/services/modeladmin/revision_resync.go | 11 ++++++- .../modeladmin/revision_resync_test.go | 15 ++++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/core/application/startup.go b/core/application/startup.go index a5aec5090..12217e426 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -267,6 +267,10 @@ func New(opts ...config.AppOption) (*Application, error) { } // Initialize distributed mode services (NATS, object storage, node registry) + // revisionStore is built inside the distributed block below but used after + // the model configs are loaded, so it is declared out here. + var revisionStore modeladmin.RevisionStore + distSvc, err := initDistributed(options, application.authDB, application.ModelConfigLoader()) if err != nil { return nil, fmt.Errorf("distributed mode initialization failed: %w", err) @@ -373,16 +377,9 @@ func New(opts ...config.AppOption) (*Application, error) { cfgLoaderOpts := options.ToConfigLoaderOptions() modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup) gs.SetModelRevisionLifecycle(modelRevisionLifecycle) - // Bring the controller's stored revisions back in line with the - // configuration on disk. An inference request may only establish a - // revision, never replace one, so a model whose stored value had - // drifted stayed unroutable until someone deleted the row. - if err := modeladmin.ResyncModelConfigRevisions(options.Context, - application.ModelConfigLoader(), - modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle), - ); err != nil { - xlog.Warn("Failed to resync model config revisions", "error", err) - } + // Captured here, used after the model configs are loaded below: the + // resync reads the loader, which is still empty at this point. + revisionStore = modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle) gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { // ApplyRemoteChange honors the op: a "delete" prunes the element // (a reload-from-path is additive and cannot drop it), anything @@ -429,6 +426,18 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Error("error loading config files", "error", err) } + // Bring the controller's stored revisions back in line with the + // configuration just loaded. An inference request may only establish a + // revision, never replace one, so a model whose stored value has drifted + // stays unroutable until something republishes it. This has to run after + // the load above: the loader is empty until then, and a resync against an + // empty loader silently reconciles nothing. + if revisionStore != nil { + if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), revisionStore); err != nil { + xlog.Warn("Failed to resync model config revisions", "error", err) + } + } + if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil { xlog.Error("error registering external backends", "error", err) } diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go index 4724f60e0..1eceff3f9 100644 --- a/core/services/modeladmin/revision_resync.go +++ b/core/services/modeladmin/revision_resync.go @@ -70,8 +70,17 @@ func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigL return nil } + configs := loader.GetAllModelsConfigs() + if len(configs) == 0 { + // Reconciling nothing is indistinguishable from reconciling correctly, + // which is how a caller that ran this before the configs were loaded + // went unnoticed. Say so rather than report success. + xlog.Warn("Skipping model config revision resync: no model configurations are loaded") + return nil + } + var transitions []ModelRevisionTransition - for _, cfg := range loader.GetAllModelsConfigs() { + for _, cfg := range configs { want, err := config.ModelConfigRevision(&cfg) if err != nil { return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go index 3979ccf75..127725442 100644 --- a/core/services/modeladmin/revision_resync_test.go +++ b/core/services/modeladmin/revision_resync_test.go @@ -140,3 +140,18 @@ var _ = Describe("ResyncModelConfigRevisions", func() { Expect(store.applied).To(BeEmpty()) }) }) + +// Running the resync before the model configs are loaded reconciled nothing +// while reporting success, which is how a mis-ordered startup call went +// unnoticed. An empty loader is now called out instead of looking like a +// clean run. +var _ = Describe("ResyncModelConfigRevisions with nothing loaded", func() { + It("does not touch stored revisions when no configs are loaded", func() { + dir := GinkgoT().TempDir() + loader := config.NewModelConfigLoader(dir) + store := &stubRevisionStore{stored: map[string]string{"served-before": "stale"}} + + Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(store.applied).To(BeEmpty()) + }) +}) From 2f625becf62ac86b075abadc23c1a4028a975760 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:31:36 +0200 Subject: [PATCH 07/75] chore(website): refresh the counters (#11697) Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- website/data/stats.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/data/stats.yaml b/website/data/stats.yaml index cee0db5e9..8bd8a0f9c 100644 --- a/website/data/stats.yaml +++ b/website/data/stats.yaml @@ -3,10 +3,10 @@ # The four GitHub fields are rewritten by .github/ci/refresh-site-counters.sh, # which runs weekly from .github/workflows/refresh-site-counters.yml. Editing # them by hand works but will be overwritten on the next run. -stars: 48067 -forks: 4320 -contributors: 225 -releases: 133 +stars: 48646 +forks: 4377 +contributors: 230 +releases: 136 # The GitHub API cannot answer for this one, so it is maintained by hand and # the refresh script carries it through untouched. From a8bc64cd09e573230048b0c87be8f5ebe6a7cfa2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 24 Aug 2026 09:32:26 +0200 Subject: [PATCH 08/75] fix(ci): bound Discord release summaries (#11695) * fix(ci): bound Discord release summaries The release model can return more than Discord's 2,000-character message limit. Discord then rejects the entire release notification. Ask the model for a smaller response and truncate extracted content to 1,800 characters before the notification step. The smaller bound leaves room below Discord's hard limit when model output varies. Assisted-by: Codex:gpt-5 * fix(tests): implement node liveness stub NodeCommandSender now requires PingNode. The endpoint test stub must implement it before the package can compile. Assisted-by: Codex:gpt-5 [Codex] --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- .github/workflows/notify-releases.yaml | 7 ++++--- core/http/endpoints/localai/nodes_backends_list_test.go | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/notify-releases.yaml b/.github/workflows/notify-releases.yaml index eab8ce54f..8711c4745 100644 --- a/.github/workflows/notify-releases.yaml +++ b/.github/workflows/notify-releases.yaml @@ -31,13 +31,14 @@ jobs: messages: [ { role: "system", - content: "Write a discord message with a bullet point summary of the release notes." + content: "Write a Discord message with a bullet point summary of the release notes. Keep the complete message under 1800 characters." }, { role: "user", content: $input } - ] + ], + max_tokens: 450 }') # Send the request to LocalAI API @@ -46,7 +47,7 @@ jobs: -d "$json_payload") # Extract the summary from the response - summary=$(echo $response | jq -r '.choices[0].message.content') + summary=$(printf '%s' "$response" | jq -er '.choices[0].message.content | strings | .[0:1800]') # Print the summary # -H "Authorization: Bearer $API_KEY" \ diff --git a/core/http/endpoints/localai/nodes_backends_list_test.go b/core/http/endpoints/localai/nodes_backends_list_test.go index c625e8e95..636ab58b8 100644 --- a/core/http/endpoints/localai/nodes_backends_list_test.go +++ b/core/http/endpoints/localai/nodes_backends_list_test.go @@ -42,6 +42,8 @@ func (s *stubNodeCommandSender) StopBackend(_, _ string) error { return nil } func (s *stubNodeCommandSender) UnloadModelOnNode(_, _ string) error { return nil } +func (s *stubNodeCommandSender) PingNode(_ string) error { return nil } + var _ = Describe("ListBackendsOnNodeEndpoint", func() { var registry *nodes.NodeRegistry From 7ff9d9942b12674cc033db04509a1078a3e1da06 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 24 Aug 2026 09:32:48 +0200 Subject: [PATCH 09/75] fix(distributed): restore node liveness tests (#11694) * fix(distributed): restore node liveness tests The router now probes models.running before it schedules work. The E2E workers only mocked backend.install, so every test node appeared offline. The endpoint test double also missed the new PingNode method and stopped the Linux, Apple, and lint jobs during compilation. Mock the existing worker reply in both distributed fixtures and keep the endpoint test double aligned with NodeCommandSender. Assisted-by: Codex:gpt-5 [golangci-lint] * fix(tests): check node liveness replies The liveness test subscriptions ignored setup and reply errors. Errcheck rejected each branch that carried them. Assisted-by: Codex:gpt-5 [golangci-lint] --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- tests/e2e/distributed/distributed_full_flow_test.go | 6 ++++++ tests/e2e/distributed/router_tracking_test.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index 5eb9ff442..ad7f2669a 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -260,6 +260,12 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() data, _ := json.Marshal(reply) msg.Respond(data) }) + _, err := infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { + data, _ := json.Marshal(messaging.ModelsRunningReply{}) + _ = msg.Respond(data) + }) + Expect(err).NotTo(HaveOccurred()) + FlushNATS(infra.NC) return router } diff --git a/tests/e2e/distributed/router_tracking_test.go b/tests/e2e/distributed/router_tracking_test.go index 9691b31b0..75895a372 100644 --- a/tests/e2e/distributed/router_tracking_test.go +++ b/tests/e2e/distributed/router_tracking_test.go @@ -66,6 +66,12 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { data, _ := json.Marshal(reply) msg.Respond(data) }) + _, err = infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { + data, _ := json.Marshal(messaging.ModelsRunningReply{}) + _ = msg.Respond(data) + }) + Expect(err).NotTo(HaveOccurred()) + FlushNATS(infra.NC) // Start a mock gRPC backend using the same helper as full flow tests llm := &trackingTestLLM{} From e470d4b625de10d55517b2220f36835bf1938885 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 24 Aug 2026 09:33:10 +0200 Subject: [PATCH 10/75] feat(gallery): add Qwen3.8 OBLITERATED variants (#11691) * feat(gallery): add Qwen3.8 OBLITERATED variants Add Q4_K_M and Q8_0 llama.cpp builds with the shared BF16 vision projector. Assisted-by: Codex:gpt-5 * fix(tests): implement node liveness stub NodeCommandSender now requires PingNode. The endpoint test stub must implement it before the package can compile. Assisted-by: Codex:gpt-5 [Codex] * fix(distributed): restore node liveness tests The router now probes models.running before it schedules work. The E2E workers only mocked backend.install, so every test node appeared offline. The endpoint test double also missed the new PingNode method and stopped the Linux, Apple, and lint jobs during compilation. Mock the existing worker reply in both distributed fixtures and keep the endpoint test double aligned with NodeCommandSender. Assisted-by: Codex:gpt-5 [golangci-lint] * fix(tests): check node liveness replies The liveness test subscriptions ignored setup and reply errors. Errcheck rejected each branch that carried them. Assisted-by: Codex:gpt-5 [golangci-lint] --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 94 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index c9a240db2..5c40b9a65 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -549,6 +549,100 @@ - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf sha256: d65001a94c4b6852bc7a0e7c5cc92fe8506755bb270e54483fd5feec7ae39a19 +- &qwen3-8-27b-obliterated + name: "qwen3.8-27b-obliterated-q4" + variants: + - model: qwen3.8-27b-obliterated-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/OBLITERATUS/Qwen3.8-27B-OBLITERATED + description: | + Qwen3.8-27B OBLITERATED is an Apache-2.0 Qwen3.8 vision-language model + modified for refusal-removal and red-team research. It retains reasoning, + coding, tool use, image, and video capabilities, but its safety guardrails + have been removed. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. The + linked variant uses the higher-quality Q8_0 model. The publisher recommends + greedy decoding with a 1.15 repetition penalty. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - uncensored + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + last_checked: "2026-08-24" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + repeat_penalty: 1.15 + temperature: 0 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf + sha256: c5e4fe705883e244a468c9e445c8d6ba37fd310b0113e25d2b8a7f2d6f1243e8 + - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf + sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 +- !!merge <<: *qwen3-8-27b-obliterated + name: "qwen3.8-27b-obliterated-q8" + variants: [] + description: | + Qwen3.8-27B OBLITERATED in the higher-quality Q8_0 GGUF format. This model + is modified for refusal-removal and red-team research, and its safety + guardrails have been removed. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + repeat_penalty: 1.15 + temperature: 0 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q8_0.gguf + sha256: 4ed72a101dfa7f8fd642598368c4d334f1334cedc5254b71a06b5c4a542c59fc + - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf + uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf + sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 - &qwen3-8-27b name: "qwen3.8-27b-q4" variants: From d7ff43781d79254b07861e29781ee97d553efd99 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:33:44 +0200 Subject: [PATCH 11/75] fix(oci): resume interrupted layer downloads (#11688) quay.io redirects blob downloads to pre-signed S3/Akamai URLs that expire after about 10 minutes. On a slow connection a multi-GiB backend layer cannot finish inside that window, so the connection drops mid-stream on every attempt. The retry added for #10577 restarted each attempt from byte zero, which replayed the same failure until the budget ran out and the install failed with "unexpected EOF". A retry now keeps the bytes already on disk and re-requests the blob with "Range: bytes=N-". Each request goes back to the registry, so it gets a fresh redirect URL and auth token. The retry budget only counts attempts that made no forward progress, so a slow link that keeps advancing keeps downloading. A resumed file is spliced from separate responses and bypasses the digest check in layer.Compressed(), so the assembled file is re-verified against the layer digest before it is trusted; on a mismatch the download starts over through the verified reader. Fixes #10577 Assisted-by: Claude Code:claude-fable-5 Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- pkg/oci/image.go | 207 +++++++++++++++++++++++--- pkg/oci/image_resume_internal_test.go | 151 +++++++++++++++++++ pkg/oci/layer_internal_test.go | 18 ++- pkg/oci/layer_resume_internal_test.go | 191 ++++++++++++++++++++++++ 4 files changed, 538 insertions(+), 29 deletions(-) create mode 100644 pkg/oci/image_resume_internal_test.go create mode 100644 pkg/oci/layer_resume_internal_test.go diff --git a/pkg/oci/image.go b/pkg/oci/image.go index 8e1546647..c44f7e5eb 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -80,27 +80,133 @@ var layerRetryBackoff = func(attempt int) time.Duration { return d } +// blobRangeOpener re-opens a layer blob at a byte offset. It returns the +// stream and the offset it actually starts at: the requested offset when the +// server honoured the Range request, or 0 when it ignored it and is sending +// the blob from the first byte again. +type blobRangeOpener func(ctx context.Context, offset int64) (io.ReadCloser, int64, error) + +// newBlobRangeOpener returns a blobRangeOpener that re-fetches the layer's +// blob from its registry with an HTTP Range request. Registries like quay.io +// redirect blob downloads to pre-signed S3/CDN URLs that expire after ~10 +// minutes; on a slow connection a multi-GiB layer cannot finish inside that +// window, so restarting from byte zero can never succeed while resuming from +// the current offset can (docker pull survives the same expiry this way). +// Each call goes back to the registry, so it obtains a fresh redirect URL and +// a fresh auth token. Returns nil when imageRef does not name a registry blob +// (e.g. local tarballs), which disables resuming. See issue #10577. +func newBlobRangeOpener(imageRef string, layer v1.Layer, auth *registrytypes.AuthConfig, base http.RoundTripper) blobRangeOpener { + ref, err := name.ParseReference(imageRef) + if err != nil { + return nil + } + digest, err := layer.Digest() + if err != nil || digest.Hex == "" { + return nil + } + repo := ref.Context() + if base == nil { + base = http.DefaultTransport + } + var authenticator authn.Authenticator + if auth != nil { + authenticator = staticAuth{auth} + } else if authenticator, err = authn.DefaultKeychain.Resolve(repo.Registry); err != nil { + authenticator = authn.Anonymous + } + blobURL := fmt.Sprintf("%s://%s/v2/%s/blobs/%s", repo.Registry.Scheme(), repo.RegistryStr(), repo.RepositoryStr(), digest.String()) + + return func(ctx context.Context, offset int64) (io.ReadCloser, int64, error) { + tr, err := transport.NewWithContext(ctx, repo.Registry, authenticator, base, []string{repo.Scope(transport.PullScope)}) + if err != nil { + return nil, 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobURL, nil) + if err != nil { + return nil, 0, err + } + if offset > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } + req.Header.Set("User-Agent", UserAgent()) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + return nil, 0, err + } + switch resp.StatusCode { + case http.StatusPartialContent: + return resp.Body, offset, nil + case http.StatusOK: + return resp.Body, 0, nil + default: + _ = resp.Body.Close() + return nil, 0, fmt.Errorf("unexpected status %d resuming blob %s", resp.StatusCode, digest.String()) + } + } +} + +// verifyLayerFile proves the assembled layer file matches the digest the +// registry advertised. A resumed download splices bytes from independent HTTP +// responses and bypasses the verified reader layer.Compressed() provides, so +// the whole file must be re-checked before it is trusted. +func verifyLayerFile(layer v1.Layer, f *os.File) error { + digest, err := layer.Digest() + if err != nil || digest.Hex == "" || digest.Algorithm != "sha256" { + return nil + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + got, _, err := v1.SHA256(f) + if err != nil { + return err + } + if got.Hex != digest.Hex { + return fmt.Errorf("resumed layer digest mismatch: got %s, want %s", got, digest) + } + return nil +} + // downloadLayerToFile streams a single compressed layer into dst, retrying on // transient network errors (unexpected EOF, connection reset, ...). Large // backend images (e.g. vLLM) are several GiB and a single dropped connection // mid-stream previously failed the whole install with "unexpected EOF" and no -// recovery. The registry transport already retries manifest fetches via -// defaultRetryPredicate (see GetImage/GetImageDigest); this extends the same -// behaviour to the layer data stream. See issue #10577. -func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, progress *progressWriter) error { +// recovery. When resume is non-nil, a retry keeps the bytes already on disk +// and continues from that offset instead of starting over: registries that +// serve blobs through expiring pre-signed URLs (quay.io + S3/Akamai) cut off +// every full-length transfer on slow connections, so restarting can never +// finish while resuming makes progress each round. The retry budget only +// counts attempts that made no forward progress, so a download that keeps +// advancing keeps going. See issue #10577. +func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, progress *progressWriter, resume blobRangeOpener) error { var lastErr error + // written tracks the valid bytes currently in dst across attempts, and + // bestWritten the furthest offset any attempt has reached: only beating + // it counts as forward progress for the retry budget, so a server that + // ignores Range requests and keeps dropping mid-stream still runs out + // of attempts instead of looping forever. + var written, bestWritten int64 + // resumed records whether any byte in dst came from a resumed raw blob + // fetch, which requires re-verifying the assembled file at the end. + resumed := false + + truncate := func() error { + if _, err := dst.Seek(0, io.SeekStart); err != nil { + return err + } + if err := dst.Truncate(0); err != nil { + return err + } + written = 0 + resumed = false + if progress != nil { + progress.written = 0 + } + return nil + } + for attempt := 0; attempt <= layerDownloadRetries; attempt++ { if attempt > 0 { - // Discard any partial data from the previous failed attempt. - if _, err := dst.Seek(0, io.SeekStart); err != nil { - return err - } - if err := dst.Truncate(0); err != nil { - return err - } - if progress != nil { - progress.written = 0 - } select { case <-ctx.Done(): return ctx.Err() @@ -108,19 +214,69 @@ func downloadLayerToFile(ctx context.Context, layer v1.Layer, dst *os.File, prog } } - var w io.Writer = dst - if progress != nil { - w = io.MultiWriter(dst, progress) + var reader io.ReadCloser + if attempt > 0 && resume != nil && written > 0 { + r, offset, rerr := resume(ctx, written) + switch { + case rerr != nil: + // Keep the partial bytes: opening the resume stream can + // fail transiently (token refresh, connection refused) + // and the next attempt can still continue from here. + lastErr = rerr + case offset != written: + // The server ignored the Range request and is sending + // the blob from the first byte: drop the partial data. + if err := truncate(); err != nil { + _ = r.Close() + return err + } + reader = r + resumed = true + default: + reader = r + resumed = true + } + } else { + // First attempt, or no way to resume: restart from scratch + // through the digest-verifying layer reader. + if err := truncate(); err != nil { + return err + } + reader, lastErr = layer.Compressed() } - var reader io.ReadCloser - reader, lastErr = layer.Compressed() - if lastErr == nil { - _, lastErr = xio.Copy(ctx, w, reader) + if reader != nil { + var w io.Writer = dst + if progress != nil { + w = io.MultiWriter(dst, progress) + } + var n int64 + n, lastErr = xio.Copy(ctx, w, reader) + written += n _ = reader.Close() + if written > bestWritten { + // Forward progress: don't charge this round against the + // retry budget, or slow links would still exhaust it. + bestWritten = written + attempt = 0 + } } + if lastErr == nil { - return nil + if !resumed { + return nil + } + verr := verifyLayerFile(layer, dst) + if verr == nil { + return nil + } + // The spliced file is corrupt: discard it and retry cleanly. + logs.Warn.Printf("discarding resumed layer download: %v", verr) + lastErr = verr + if err := truncate(); err != nil { + return err + } + continue } // Stop early on context cancellation or non-retryable errors. @@ -382,8 +538,11 @@ func DownloadOCIImageTar(ctx context.Context, img v1.Image, imageRef string, tar } } - // Download the compressed layer, retrying on transient network errors. - err = downloadLayerToFile(ctx, layer, file, progress) + // Download the compressed layer, retrying on transient network + // errors and resuming from the last byte received where possible. + // Anonymous/default-keychain credentials match what GetImage uses + // for every in-tree caller (they all pass a nil auth). + err = downloadLayerToFile(ctx, layer, file, progress, newBlobRangeOpener(imageRef, layer, nil, nil)) file.Close() if err != nil { return fmt.Errorf("failed to download layer %d: %v", i, err) diff --git a/pkg/oci/image_resume_internal_test.go b/pkg/oci/image_resume_internal_test.go new file mode 100644 index 000000000..dec545d96 --- /dev/null +++ b/pkg/oci/image_resume_internal_test.go @@ -0,0 +1,151 @@ +package oci + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// droppingBlobRegistry emulates how quay.io serves layer blobs from S3/Akamai +// with a short-lived pre-signed URL: a full-blob GET on a slow connection is +// always cut off mid-transfer, so a client that restarts from byte zero can +// never complete the download. Only a client that resumes with a Range request +// (like docker pull does) receives the remaining bytes and can finish. +type droppingBlobRegistry struct { + inner http.Handler + + mu sync.Mutex + rangeRequests []int64 + fullRequests int +} + +// dropThreshold separates real layer blobs from small metadata blobs (image +// config), which are served untouched. +const dropThreshold = 1024 + +func (h *droppingBlobRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || !strings.Contains(r.URL.Path, "/blobs/sha256:") { + h.inner.ServeHTTP(w, r) + return + } + + // Fetch the full blob from the inner registry (which does not speak + // Range) and apply the Range semantics here. + inner := r.Clone(r.Context()) + inner.Header.Del("Range") + rec := httptest.NewRecorder() + h.inner.ServeHTTP(rec, inner) + body := rec.Body.Bytes() + if rec.Code != http.StatusOK || len(body) <= dropThreshold { + for k, vv := range rec.Header() { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(rec.Code) + _, _ = w.Write(body) + return + } + + if rh := r.Header.Get("Range"); rh != "" { + offset, err := strconv.ParseInt(strings.TrimSuffix(strings.TrimPrefix(rh, "bytes="), "-"), 10, 64) + if err != nil || offset < 0 || offset >= int64(len(body)) { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + h.mu.Lock() + h.rangeRequests = append(h.rangeRequests, offset) + h.mu.Unlock() + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, len(body)-1, len(body))) + w.Header().Set("Content-Length", strconv.Itoa(len(body)-int(offset))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(body[offset:]) + return + } + + h.mu.Lock() + h.fullRequests++ + h.mu.Unlock() + + // Announce the full size but deliver only half, then sever the + // connection, like a pre-signed URL expiring mid-download. + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body[:len(body)/2]) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) +} + +var _ = Describe("DownloadOCIImageTar resume", func() { + var ( + server *httptest.Server + reg *droppingBlobRegistry + tmpDir string + restoreWait func() + ) + + BeforeEach(func() { + reg = &droppingBlobRegistry{inner: registry.New()} + server = httptest.NewServer(reg) + + var err error + tmpDir, err = os.MkdirTemp("", "oci-resume-e2e-*") + Expect(err).NotTo(HaveOccurred()) + + prev := layerRetryBackoff + layerRetryBackoff = func(int) time.Duration { return 0 } + restoreWait = func() { layerRetryBackoff = prev } + }) + + AfterEach(func() { + restoreWait() + server.Close() + _ = os.RemoveAll(tmpDir) + }) + + It("completes the download by resuming interrupted layer transfers with Range requests", func() { + img, err := random.Image(4096, 1) + Expect(err).NotTo(HaveOccurred()) + + imageRef := strings.TrimPrefix(server.URL, "http://") + "/testrepo/backend:latest" + ref, err := name.ParseReference(imageRef) + Expect(err).NotTo(HaveOccurred()) + Expect(remote.Write(ref, img)).To(Succeed()) + + pulled, err := GetImage(imageRef, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + + tarPath := filepath.Join(tmpDir, "image.tar") + err = DownloadOCIImageTar(context.Background(), pulled, imageRef, tarPath, nil) + Expect(err).NotTo(HaveOccurred()) + + // The full-blob attempt was cut off, so success is only possible + // through at least one Range request picking up where it stopped. + reg.mu.Lock() + defer reg.mu.Unlock() + Expect(reg.rangeRequests).NotTo(BeEmpty()) + for _, off := range reg.rangeRequests { + Expect(off).To(BeNumerically(">", 0)) + } + + fi, err := os.Stat(tarPath) + Expect(err).NotTo(HaveOccurred()) + Expect(fi.Size()).To(BeNumerically(">", 0)) + }) +}) diff --git a/pkg/oci/layer_internal_test.go b/pkg/oci/layer_internal_test.go index faa8d5a45..23ba48160 100644 --- a/pkg/oci/layer_internal_test.go +++ b/pkg/oci/layer_internal_test.go @@ -33,14 +33,18 @@ func (r *failingReader) Read(p []byte) (int, error) { // fakeLayer is a minimal v1.Layer whose Compressed() fails failUntil times with // err (after emitting a partial prefix) before finally returning data in full. +// The failing attempts emit prefix when set, or placeholder garbage otherwise. +// digest, when set, is what Digest() reports. type fakeLayer struct { data []byte + prefix []byte + digest v1.Hash failUntil int err error calls int } -func (f *fakeLayer) Digest() (v1.Hash, error) { return v1.Hash{}, nil } +func (f *fakeLayer) Digest() (v1.Hash, error) { return f.digest, nil } func (f *fakeLayer) DiffID() (v1.Hash, error) { return v1.Hash{}, nil } func (f *fakeLayer) Size() (int64, error) { return int64(len(f.data)), nil } func (f *fakeLayer) MediaType() (types.MediaType, error) { return types.DockerLayer, nil } @@ -51,7 +55,11 @@ func (f *fakeLayer) Uncompressed() (io.ReadCloser, error) { func (f *fakeLayer) Compressed() (io.ReadCloser, error) { f.calls++ if f.calls <= f.failUntil { - return io.NopCloser(&failingReader{prefix: []byte("partial-garbage"), err: f.err}), nil + prefix := f.prefix + if prefix == nil { + prefix = []byte("partial-garbage") + } + return io.NopCloser(&failingReader{prefix: prefix, err: f.err}), nil } return io.NopCloser(bytes.NewReader(f.data)), nil } @@ -86,7 +94,7 @@ var _ = Describe("downloadLayerToFile", func() { err: io.ErrUnexpectedEOF, } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(layer.calls).To(Equal(3)) @@ -104,7 +112,7 @@ var _ = Describe("downloadLayerToFile", func() { err: errors.New("permission denied"), } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).To(HaveOccurred()) Expect(layer.calls).To(Equal(1)) }) @@ -116,7 +124,7 @@ var _ = Describe("downloadLayerToFile", func() { err: io.ErrUnexpectedEOF, } - err := downloadLayerToFile(context.Background(), layer, dst, nil) + err := downloadLayerToFile(context.Background(), layer, dst, nil, nil) Expect(err).To(MatchError(io.ErrUnexpectedEOF)) Expect(layer.calls).To(Equal(layerDownloadRetries + 1)) }) diff --git a/pkg/oci/layer_resume_internal_test.go b/pkg/oci/layer_resume_internal_test.go new file mode 100644 index 000000000..3f9e032d9 --- /dev/null +++ b/pkg/oci/layer_resume_internal_test.go @@ -0,0 +1,191 @@ +package oci + +import ( + "bytes" + "context" + "io" + "os" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// recordingOpener is a test blobRangeOpener that records the offsets it was +// asked to resume from and delegates the stream to open. +type recordingOpener struct { + offsets []int64 + open func(offset int64) (io.ReadCloser, int64, error) +} + +func (o *recordingOpener) opener() blobRangeOpener { + return func(_ context.Context, offset int64) (io.ReadCloser, int64, error) { + o.offsets = append(o.offsets, offset) + return o.open(offset) + } +} + +func sha256Of(data []byte) v1.Hash { + h, _, err := v1.SHA256(bytes.NewReader(data)) + Expect(err).NotTo(HaveOccurred()) + return h +} + +var _ = Describe("downloadLayerToFile resume", func() { + var ( + dst *os.File + data []byte + restoreWait func() + ) + + readDst := func() string { + got, err := os.ReadFile(dst.Name()) + Expect(err).NotTo(HaveOccurred()) + return string(got) + } + + BeforeEach(func() { + var err error + dst, err = os.CreateTemp("", "layer-resume-*.tar.gz") + Expect(err).NotTo(HaveOccurred()) + + data = []byte("0123456789abcdefghijklmnopqrstuvwxyzABCD") + + prev := layerRetryBackoff + layerRetryBackoff = func(int) time.Duration { return 0 } + restoreWait = func() { layerRetryBackoff = prev } + }) + + AfterEach(func() { + restoreWait() + _ = dst.Close() + _ = os.Remove(dst.Name()) + }) + + It("continues from the interruption offset instead of restarting", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(data[offset:])), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + Expect(readDst()).To(Equal(string(data))) + // The interrupted first attempt left 15 bytes; the resume must ask + // for exactly the rest, without a second full-stream attempt. + Expect(rec.offsets).To(Equal([]int64{15})) + Expect(layer.calls).To(Equal(1)) + }) + + It("restarts cleanly when the server ignores the Range request", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(int64) (io.ReadCloser, int64, error) { + // A 200 response: the whole blob from the first byte. + return io.NopCloser(bytes.NewReader(data)), 0, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + // The partial bytes must have been discarded, not prepended. + Expect(readDst()).To(Equal(string(data))) + Expect(rec.offsets).To(HaveLen(1)) + Expect(layer.calls).To(Equal(1)) + }) + + It("discards a resumed download whose digest does not match", func() { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + corrupt := bytes.Repeat([]byte("x"), len(data)-int(offset)) + return io.NopCloser(bytes.NewReader(corrupt)), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + // The spliced file failed verification, so the download must have + // started over through the verified layer reader and succeeded. + Expect(readDst()).To(Equal(string(data))) + Expect(rec.offsets).To(Equal([]int64{15})) + Expect(layer.calls).To(Equal(2)) + }) + + It("keeps retrying beyond the budget while each resume makes progress", func() { + const step = 5 + layer := &fakeLayer{ + data: data, + prefix: data[:step], + digest: sha256Of(data), + failUntil: 1, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + if offset+step >= int64(len(data)) { + return io.NopCloser(bytes.NewReader(data[offset:])), offset, nil + } + return io.NopCloser(&failingReader{prefix: data[offset : offset+step], err: io.ErrUnexpectedEOF}), offset, nil + }} + + err := downloadLayerToFile(context.Background(), layer, dst, nil, rec.opener()) + Expect(err).NotTo(HaveOccurred()) + Expect(readDst()).To(Equal(string(data))) + // 40 bytes delivered 5 at a time: 7 resumes, far more rounds than + // the retry budget allows for stalled attempts. + Expect(len(rec.offsets)).To(BeNumerically(">", layerDownloadRetries)) + }) + + It("gives up when resumes stop making progress", func(ctx SpecContext) { + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1000, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(offset int64) (io.ReadCloser, int64, error) { + // Resume accepted but the connection dies before any byte. + return io.NopCloser(&failingReader{err: io.ErrUnexpectedEOF}), offset, nil + }} + + err := downloadLayerToFile(ctx, layer, dst, nil, rec.opener()) + Expect(err).To(MatchError(io.ErrUnexpectedEOF)) + Expect(len(rec.offsets)).To(Equal(layerDownloadRetries)) + }, NodeTimeout(10*time.Second)) + + It("terminates when the server ignores Range and keeps dropping mid-stream", func(ctx SpecContext) { + // Each round delivers some bytes from the start and dies: the file + // never gets further than before, so this must exhaust the budget + // rather than count the repeated partial bytes as progress. + layer := &fakeLayer{ + data: data, + prefix: data[:15], + digest: sha256Of(data), + failUntil: 1000, + err: io.ErrUnexpectedEOF, + } + rec := &recordingOpener{open: func(int64) (io.ReadCloser, int64, error) { + return io.NopCloser(&failingReader{prefix: data[:15], err: io.ErrUnexpectedEOF}), 0, nil + }} + + err := downloadLayerToFile(ctx, layer, dst, nil, rec.opener()) + Expect(err).To(MatchError(io.ErrUnexpectedEOF)) + Expect(len(rec.offsets)).To(Equal(layerDownloadRetries)) + }, NodeTimeout(10*time.Second)) +}) From 1bee6b14b7a6fe46289c4ec79336a6a9fa8fb286 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:33:56 +0200 Subject: [PATCH 12/75] chore: :arrow_up: Update CrispStrobe/CrispASR to `ae4474dd8306384a0e697183d863dfc52e69a2fb` (#11684) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 9cf913762..b87b395a7 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=74bb374a8cc74284348d76a0a6e944180fbe6b07 +CRISPASR_VERSION?=ae4474dd8306384a0e697183d863dfc52e69a2fb SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From dc0961f9620bcccc8dae721e38810d613b61dbe6 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:34:09 +0200 Subject: [PATCH 13/75] chore: :arrow_up: Update 0xShug0/audio.cpp to `288a2712316470847a730e55db9ac9e5062a2b03` (#11683) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index feb7ba0a0..bee144fbc 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=4d383be1bff107e823ffc19120dcb6c78d493c0f +AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 336b97fcfe31557c06a2b677bf1aaecf8762cb82 Mon Sep 17 00:00:00 2001 From: DanielSwift1992 <40451130+DanielSwift1992@users.noreply.github.com> Date: Mon, 24 Aug 2026 03:44:36 -0400 Subject: [PATCH 14/75] chore(deps): remove 16 dependabot entries for directories that no longer exist (#11686) Remove 16 dependabot entries for directories that no longer exist Signed-off-by: Daniil S --- .github/dependabot.yml | 66 +----------------------------------------- 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17e85e101..cefd0cefd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,10 +29,6 @@ updates: schedule: # Check for updates to GitHub Actions every weekday interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/bark" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/common/template" schedule: @@ -55,30 +51,10 @@ updates: ignore: - dependency-name: "torch" - dependency-name: "transformers" - - package-ecosystem: "pip" - directory: "/backend/python/exllama" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/exllama2" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/mamba" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/openvoice" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/rerankers" schedule: interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/sentencetransformers" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/transformers" schedule: @@ -86,44 +62,4 @@ updates: - package-ecosystem: "pip" directory: "/backend/python/vllm" schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/chainlit" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/functions" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain/langchainpy-localai-example" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain-chroma" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/streamlit-bot" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/k8sgpt" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/kubernetes" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/langchain" - schedule: - interval: "weekly" - - package-ecosystem: "gomod" - directory: "/examples/semantic-todo" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/telegram-bot" - schedule: - interval: "weekly" + interval: "weekly" \ No newline at end of file From dc303aa96c6e103160126c931d9684ddf9d3202f Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:44:51 +0200 Subject: [PATCH 15/75] feat(swagger): update swagger (#11682) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- swagger/docs.go | 21 +++++++++++++++++++++ swagger/swagger.json | 21 +++++++++++++++++++++ swagger/swagger.yaml | 14 ++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/swagger/docs.go b/swagger/docs.go index b399f73e8..16fa1de85 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -4580,6 +4580,9 @@ const docTemplate = `{ "type": "object", "properties": { "config": {}, + "config_revision": { + "type": "string" + }, "details": { "type": "array", "items": { @@ -4595,6 +4598,9 @@ const docTemplate = `{ "message": { "type": "string" }, + "pending_cleanup": { + "type": "integer" + }, "success": { "type": "boolean" } @@ -4735,9 +4741,24 @@ const docTemplate = `{ "description": "e.g. \"llama-cpp\"; used by reconciler to replicate loads", "type": "string" }, + "cleanup_attempts": { + "type": "integer" + }, + "cleanup_error": { + "type": "string" + }, + "cleanup_next_retry_at": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, "created_at": { "type": "string" }, + "effective_options_hash": { + "type": "string" + }, "id": { "type": "string" }, diff --git a/swagger/swagger.json b/swagger/swagger.json index b04ebca6d..2be8aea42 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -4577,6 +4577,9 @@ "type": "object", "properties": { "config": {}, + "config_revision": { + "type": "string" + }, "details": { "type": "array", "items": { @@ -4592,6 +4595,9 @@ "message": { "type": "string" }, + "pending_cleanup": { + "type": "integer" + }, "success": { "type": "boolean" } @@ -4732,9 +4738,24 @@ "description": "e.g. \"llama-cpp\"; used by reconciler to replicate loads", "type": "string" }, + "cleanup_attempts": { + "type": "integer" + }, + "cleanup_error": { + "type": "string" + }, + "cleanup_next_retry_at": { + "type": "string" + }, + "config_revision": { + "type": "string" + }, "created_at": { "type": "string" }, + "effective_options_hash": { + "type": "string" + }, "id": { "type": "string" }, diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index f0e40bbe0..58653861b 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -414,6 +414,8 @@ definitions: localai.ModelResponse: properties: config: {} + config_revision: + type: string details: items: type: string @@ -424,6 +426,8 @@ definitions: type: string message: type: string + pending_cleanup: + type: integer success: type: boolean type: object @@ -518,8 +522,18 @@ definitions: backend_type: description: e.g. "llama-cpp"; used by reconciler to replicate loads type: string + cleanup_attempts: + type: integer + cleanup_error: + type: string + cleanup_next_retry_at: + type: string + config_revision: + type: string created_at: type: string + effective_options_hash: + type: string id: type: string in_flight: From 98649d775e552f7ac36b620256278f42ecf6d683 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:45:32 +0200 Subject: [PATCH 16/75] chore(model gallery): :robot: add 1 new models via gallery agent (#11692) chore(model gallery): :robot: add new models via gallery agent Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 5c40b9a65..c980a79fd 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,52 @@ --- +- name: "qwen3.8-27b-dflash2" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF + description: | + # Qwen3.8-27B + + > [!Note] + > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format. + > + > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc. + + > [!Tip] + > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud. + > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates. + + Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date. + + ... + license: "apache-2.0" + tags: + - llm + - gguf + icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + presence_penalty: 1.5 + repeat_penalty: 1 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + sha256: 18a380efc9b7ed8d88677fc895f5c11ae170653434ee378f7348f715c14d0594 + uri: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf - name: "huihui-qwen3.8-27b-abliterated" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From 505a6d040b663930c0afdde3435d7af410dbe978 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 09:05:50 +0000 Subject: [PATCH 17/75] fix(distributed): publish the revision a request actually carries Two code paths computed a model's revision. Inference resolves the config through the loader, which applies SetDefaults a second time. Everything that publishes a revision hashed the stored config instead, with SetDefaults applied once. SetDefaults is not idempotent for every model: it re-runs the GGUF guess and the hardware defaults, both of which read state the stored config does not carry. Where the two disagree, a publisher wrote a revision no request would ever carry, and the model became unroutable the moment it was published. On this cluster the startup resync republished one such value and every request for that model was then rejected against it. The publishers now resolve the revision through the loader, exactly as a request does, so there is one definition rather than two that agree only when SetDefaults happens to be idempotent. This covers the startup resync, a saved config edit, and enabling or disabling a model. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/application/startup.go | 2 +- core/services/modeladmin/config.go | 16 +++++++++--- core/services/modeladmin/revision_resync.go | 18 ++++++++++--- .../modeladmin/revision_resync_test.go | 25 +++++++++++-------- core/services/modeladmin/state.go | 17 +++++++++---- 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/core/application/startup.go b/core/application/startup.go index 12217e426..abc2f4a17 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -433,7 +433,7 @@ func New(opts ...config.AppOption) (*Application, error) { // the load above: the loader is empty until then, and a resync against an // empty loader silently reconciles nothing. if revisionStore != nil { - if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), revisionStore); err != nil { + if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), options, revisionStore); err != nil { xlog.Warn("Failed to resync model config revisions", "error", err) } } diff --git a/core/services/modeladmin/config.go b/core/services/modeladmin/config.go index 23de357aa..515d014f7 100644 --- a/core/services/modeladmin/config.go +++ b/core/services/modeladmin/config.go @@ -177,13 +177,21 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[ if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil { return fmt.Errorf("reload configs: %w", err) } - loaded, ok := s.Loader.GetModelConfig(updated.Name) - if !ok { + if _, ok := s.Loader.GetModelConfig(updated.Name); !ok { return fmt.Errorf("reload configs: model %q missing", updated.Name) } - revision, err := config.ModelConfigRevision(&loaded) + // Resolve the revision the way an inference request does. Hashing the + // stored config instead publishes a value no request will ever carry, + // because SetDefaults runs again on the request path and is not + // idempotent for every model, and the edit would leave the model + // unroutable. + resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(updated.Name, s.AppConfig) if err != nil { - return fmt.Errorf("compute config revision: %w", err) + return fmt.Errorf("resolve config revision: %w", err) + } + revision := resolved.PersistedConfigRevision() + if revision == "" { + return fmt.Errorf("no config revision stamped for %q", updated.Name) } _ = s.Loader.Preload(s.modelsPath()) pending, err := s.applyRevision(ctx, name, updated.Name, revision, updated.IsDisabled()) diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go index 1eceff3f9..5d84831a2 100644 --- a/core/services/modeladmin/revision_resync.go +++ b/core/services/modeladmin/revision_resync.go @@ -65,8 +65,8 @@ func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) R // A model with no stored revision is left alone. It has never been served, and // inventing controller state for it here would quarantine nothing and describe // a model that may never be requested. -func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, store RevisionStore) error { - if loader == nil || store == nil { +func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, appConfig *config.ApplicationConfig, store RevisionStore) error { + if loader == nil || store == nil || appConfig == nil { return nil } @@ -81,9 +81,19 @@ func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigL var transitions []ModelRevisionTransition for _, cfg := range configs { - want, err := config.ModelConfigRevision(&cfg) + // Resolve the revision the way an inference request does, through the + // loader, rather than hashing the stored config directly. SetDefaults + // is applied again on that path and is not idempotent for every model + // (it re-runs the GGUF guess and hardware defaults), so hashing the + // stored config yields a value no request will ever carry, and + // publishing it would wedge the model this resync exists to unwedge. + resolved, err := loader.LoadModelConfigFileByNameDefaultOptions(cfg.Name, appConfig) if err != nil { - return fmt.Errorf("compute config revision for %q: %w", cfg.Name, err) + return fmt.Errorf("resolve config for %q: %w", cfg.Name, err) + } + want := resolved.PersistedConfigRevision() + if want == "" { + return fmt.Errorf("no config revision stamped for %q", cfg.Name) } stored, err := store.GetModelConfigRevision(ctx, cfg.Name) diff --git a/core/services/modeladmin/revision_resync_test.go b/core/services/modeladmin/revision_resync_test.go index 127725442..43aceb683 100644 --- a/core/services/modeladmin/revision_resync_test.go +++ b/core/services/modeladmin/revision_resync_test.go @@ -55,12 +55,13 @@ var _ = Describe("ResyncModelConfigRevisions", func() { Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) } + // revisionOf resolves the revision the way an inference request does, which + // is the value the resync must publish. revisionOf := func(name string) string { - cfg, ok := loader.GetModelConfig(name) - Expect(ok).To(BeTrue()) - rev, err := config.ModelConfigRevision(&cfg) + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig) Expect(err).ToNot(HaveOccurred()) - return rev + Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty()) + return cfg.PersistedConfigRevision() } BeforeEach(func() { @@ -80,7 +81,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.stored["drifted"] = "a-revision-from-an-earlier-build" - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(HaveLen(1)) Expect(store.applied[0].ModelName).To(Equal("drifted")) @@ -92,7 +93,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.stored["agreed"] = revisionOf("agreed") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing") }) @@ -104,7 +105,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { write("never-served", "name: never-served\nbackend: llama-cpp\n") load() - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty()) }) @@ -116,7 +117,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { store.stored["drifted"] = "stale" store.stored["agreed"] = revisionOf("agreed") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(HaveLen(1)) Expect(store.applied[0].ModelName).To(Equal("drifted")) @@ -128,7 +129,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { store.stored["drifted"] = "stale" store.applyEr = errors.New("database is down") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed()) }) It("skips a model whose stored revision cannot be read rather than guessing", func() { @@ -136,7 +137,7 @@ var _ = Describe("ResyncModelConfigRevisions", func() { load() store.getErr = errors.New("connection reset") - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).ToNot(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed()) Expect(store.applied).To(BeEmpty()) }) }) @@ -149,9 +150,11 @@ var _ = Describe("ResyncModelConfigRevisions with nothing loaded", func() { It("does not touch stored revisions when no configs are loaded", func() { dir := GinkgoT().TempDir() loader := config.NewModelConfigLoader(dir) + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} store := &stubRevisionStore{stored: map[string]string{"served-before": "stale"}} - Expect(ResyncModelConfigRevisions(context.Background(), loader, store)).To(Succeed()) + Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed()) Expect(store.applied).To(BeEmpty()) }) }) diff --git a/core/services/modeladmin/state.go b/core/services/modeladmin/state.go index 5d87f6675..4f84b5859 100644 --- a/core/services/modeladmin/state.go +++ b/core/services/modeladmin/state.go @@ -7,7 +7,6 @@ import ( "gopkg.in/yaml.v3" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/utils" ) @@ -61,13 +60,21 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil { return fmt.Errorf("reload configs: %w", err) } - loaded, ok := s.Loader.GetModelConfig(name) - if !ok { + if _, ok := s.Loader.GetModelConfig(name); !ok { return fmt.Errorf("reload configs: model %q missing", name) } - revision, err := config.ModelConfigRevision(&loaded) + // Resolve the revision the way an inference request does. Hashing the + // stored config instead publishes a value no request will ever carry, + // because SetDefaults runs again on the request path and is not + // idempotent for every model, and the edit would leave the model + // unroutable. + resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(name, s.AppConfig) if err != nil { - return fmt.Errorf("compute config revision: %w", err) + return fmt.Errorf("resolve config revision: %w", err) + } + revision := resolved.PersistedConfigRevision() + if revision == "" { + return fmt.Errorf("no config revision stamped for %q", name) } pending, err := s.applyRevision(ctx, name, name, revision, action == ActionDisable) if err != nil { From df1a40f9c0383c5e848defaa186b7476372e8d23 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 11:45:32 +0000 Subject: [PATCH 18/75] fix(distributed): hash the config as persisted, not as defaulted The revision was computed after SetDefaults, which folds in things that are not persisted configuration: the GGUF guess, the hardware defaults, and app-level options such as threads. The GGUF guess is the damaging one. It parses the model file to fill in values like context size, and when that parse fails it falls back to a different default. Whether a multi-gigabyte file on network storage parses at a given moment is not a property of the configuration, so one unchanged YAML produced two different revisions depending on when it was read. The controller rejected every request carrying the other one, and the model stayed unroutable until the stored value happened to match again. This is why it never reproduced against a model directory with no weights in it: the guess is skipped there and both values agree. The app-level defaults are the same class of bug with a slower fuse: changing threads in the settings UI changed every model's revision and made every model unroutable. The revision is now stamped when the file is parsed, before any defaults are applied, so it is a function of the file alone. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/config/model_config_loader.go | 30 ++++++--- .../model_config_revision_stability_test.go | 67 +++++++++++++++++-- .../request_config_revision_test.go | 4 +- 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go index 4c95a9665..2a062c39d 100644 --- a/core/config/model_config_loader.go +++ b/core/config/model_config_loader.go @@ -168,6 +168,14 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model if err := yaml.Unmarshal(f, &configs); err == nil && len(configs) > 0 { for _, cc := range configs { cc.modelConfigFile = file + // Stamp before SetDefaults: the revision describes what is on disk. + // SetDefaults folds in the GGUF guess, hardware defaults and + // app-level options, none of which are persisted configuration, and + // the GGUF guess in particular depends on whether the model file + // parses at that moment. + if err := cc.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", cc.Name, err) + } cc.SetDefaults(opts...) cc.syncKnownUsecasesFromString() } @@ -182,6 +190,9 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model c.modelConfigFile = file c.syncKnownUsecasesFromString() + if err := c.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", c.Name, err) + } c.SetDefaults(opts...) return []*ModelConfig{c}, nil @@ -218,17 +229,18 @@ func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath str } } - cfg.SetDefaults(append(opts, ModelPath(modelPath))...) - - // Stamp the revision here, at the boundary between the persisted - // configuration and the request that is about to override parts of it. - // Everything downstream of this point (the request middleware) merges - // per-request prediction parameters into cfg, so a revision computed later - // would identify the request rather than the configuration. - if err := cfg.StampPersistedConfigRevision(); err != nil { - return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + // Stamp before SetDefaults, and only when this config did not come from + // disk already carrying one (a name with no config file on disk is + // synthesized above). Re-stamping a loaded config here would hash it after + // SetDefaults and reintroduce the dependency on the GGUF guess. + if cfg.PersistedConfigRevision() == "" { + if err := cfg.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + } } + cfg.SetDefaults(append(opts, ModelPath(modelPath))...) + return cfg, nil } diff --git a/core/config/model_config_revision_stability_test.go b/core/config/model_config_revision_stability_test.go index 19a0d8590..b353f3d14 100644 --- a/core/config/model_config_revision_stability_test.go +++ b/core/config/model_config_revision_stability_test.go @@ -73,18 +73,75 @@ template: }) // The request pipeline reloads the config through LoadModelConfigFileByName, - // which applies SetDefaults a second time. That must not move the revision - // away from the one model administration publishes from the loader map. + // which applies SetDefaults a second time. The stamp is taken before those + // defaults, so both the stored config and the one a request resolves carry + // the same revision. It("survives the extra SetDefaults the request path applies", func() { loader := config.NewModelConfigLoader(dir) Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) stored, ok := loader.GetModelConfig("example") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&stored) - Expect(err).ToNot(HaveOccurred()) + Expect(stored.PersistedConfigRevision()).ToNot(BeEmpty()) requestCfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) Expect(err).ToNot(HaveOccurred()) - Expect(requestCfg.PersistedConfigRevision()).To(Equal(adminRevision)) + Expect(requestCfg.PersistedConfigRevision()).To(Equal(stored.PersistedConfigRevision())) + }) +}) + +// The revision must describe the configuration as persisted, and nothing else. +// SetDefaults folds in values that are not persisted config: the GGUF guess +// (which reads the model file and can fail on slow or remote storage), the +// hardware defaults, and app-level options like threads. Hashing after that +// made the revision a function of whether a multi-gigabyte file happened to +// parse, so one unchanged YAML produced two different revisions depending on +// the moment, and the controller rejected every request carrying the other one. +var _ = Describe("Model config revision independence from runtime defaults", func() { + It("does not change when SetDefaults is applied", func() { + dir := GinkgoT().TempDir() + body := "backend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\n" + + "mmproj: llama-cpp/mmproj/example/mmproj.gguf\nname: example\n" + + "parameters:\n model: llama-cpp/models/example/example.gguf\n" + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), []byte(body), 0o600)).To(Succeed()) + + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + + stored, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + before := stored.PersistedConfigRevision() + Expect(before).ToNot(BeEmpty()) + + // Applying defaults again is what the request path does. + stored.SetDefaults(appConfig.ToConfigLoaderOptions()...) + Expect(stored.PersistedConfigRevision()).To(Equal(before)) + + resolved, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.PersistedConfigRevision()).To(Equal(before), + "the request path must carry the same revision as the stored config") + }) + + It("does not change when app-level defaults differ", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), + []byte("name: example\nbackend: llama-cpp\nparameters:\n model: m.gguf\n"), 0o600)).To(Succeed()) + + revWith := func(threads int, f16 bool) string { + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + appConfig.Threads = threads + appConfig.F16 = f16 + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + return cfg.PersistedConfigRevision() + } + + Expect(revWith(8, false)).To(Equal(revWith(1, true)), + "an operator changing threads must not make every model unroutable") }) }) diff --git a/core/http/middleware/request_config_revision_test.go b/core/http/middleware/request_config_revision_test.go index 419ae8e04..dec7bc04d 100644 --- a/core/http/middleware/request_config_revision_test.go +++ b/core/http/middleware/request_config_revision_test.go @@ -115,8 +115,8 @@ var _ = Describe("Model config revision seen by inference requests", func() { Expect(admin.LoadModelConfigsFromPath(modelDir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) loaded, ok := admin.GetModelConfig("test-model") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&loaded) - Expect(err).ToNot(HaveOccurred()) + adminRevision := loaded.PersistedConfigRevision() + Expect(adminRevision).ToNot(BeEmpty()) Expect(revisionFor(`{"model":"test-model","temperature":0.7,"messages":[{"role":"user","content":"hi"}]}`)). To(Equal(adminRevision)) From bebd812e7dbf520cfcc620135cb106d1cca1ec8f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 12:48:23 +0000 Subject: [PATCH 19/75] fix(distributed): stop flapping agent nodes on backend listing Only backend workers subscribe to backend.list. ListBackends asked every node that was not pending, offline or draining, so an agent worker could only answer "no responders", which the error handling reads as a node that has gone away. Every poll of the backends view therefore marked each agent node unhealthy, and its next heartbeat marked it healthy again. While unhealthy the node is not schedulable, so this also cost agent capacity for as long as each flap lasted. Skip non-backend workers, as the backend-op fan-out already does for the same reason. A backend worker that does not answer is still marked unhealthy: that one really is gone. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- .../nodes/managers_agent_node_test.go | 84 +++++++++++++++++++ core/services/nodes/managers_distributed.go | 13 ++- 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 core/services/nodes/managers_agent_node_test.go diff --git a/core/services/nodes/managers_agent_node_test.go b/core/services/nodes/managers_agent_node_test.go new file mode 100644 index 000000000..8ee95c083 --- /dev/null +++ b/core/services/nodes/managers_agent_node_test.go @@ -0,0 +1,84 @@ +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Agent workers do not subscribe to the backend.* subjects, so asking one to +// list its backends can only answer "no responders". ListBackends read that as +// a node that had gone away and marked it unhealthy; the node's next heartbeat +// marked it healthy again. Every poll of the backends view therefore flapped +// every agent node in the cluster, and while it was unhealthy the router would +// not schedule onto it. +var _ = Describe("Backend listing across mixed node types", func() { + var ( + db *gorm.DB + registry *NodeRegistry + mc *scriptedMessagingClient + mgr *DistributedBackendManager + ctx context.Context + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + mc = newScriptedMessagingClient() + mgr = &DistributedBackendManager{ + local: stubLocalBackendManager{}, + adapter: NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute), + registry: registry, + } + ctx = context.Background() + }) + + register := func(name, nodeType string) *BackendNode { + node := &BackendNode{Name: name, NodeType: nodeType, Address: name + ":50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + fetched, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + Expect(fetched.Status).To(Equal(StatusHealthy)) + return fetched + } + + statusOf := func(id string) string { + n, err := registry.Get(ctx, id) + Expect(err).ToNot(HaveOccurred()) + return n.Status + } + + It("leaves an agent node healthy instead of flapping it", func() { + agent := register("agent-worker-1", NodeTypeAgent) + mc.scriptNoResponders(messaging.SubjectNodeBackendList(agent.ID)) + + _, err := mgr.ListBackends() + Expect(err).ToNot(HaveOccurred()) + + Expect(statusOf(agent.ID)).To(Equal(StatusHealthy), + "an agent node cannot answer backend.list and must not be judged on it") + }) + + It("still marks a backend node unhealthy when it does not answer", func() { + backendNode := register("worker-a", NodeTypeBackend) + mc.scriptNoResponders(messaging.SubjectNodeBackendList(backendNode.ID)) + + _, err := mgr.ListBackends() + Expect(err).ToNot(HaveOccurred()) + + Expect(statusOf(backendNode.ID)).To(Equal(StatusUnhealthy), + "a backend worker that does not answer is genuinely gone") + }) +}) diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 127425b1a..4132eca79 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -331,8 +331,9 @@ func (d *DistributedBackendManager) DeleteBackendDetailed(ctx context.Context, n // populated from the first node seen so single-node-minded callers still work. // // Pending/offline/draining nodes are skipped because they aren't expected to -// answer NATS requests; unhealthy nodes are still queried — ErrNoResponders -// then marks them unhealthy and the loop continues. +// answer NATS requests, and so are non-backend workers, which do not subscribe +// to backend.list at all; unhealthy backend nodes are still queried — +// ErrNoResponders then marks them unhealthy and the loop continues. func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, error) { result := make(gallery.SystemBackends) allNodes, err := d.registry.List(context.Background()) @@ -344,6 +345,14 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining { continue } + // Only backend workers subscribe to backend.list. Asking an agent + // worker can only answer "no responders", which the error handling + // below reads as a node that has gone away, so every poll of this view + // marked every agent node unhealthy and its next heartbeat marked it + // healthy again. The backend-op fan-out skips them for the same reason. + if node.NodeType != "" && node.NodeType != NodeTypeBackend { + continue + } reply, err := d.adapter.ListBackends(node.ID) if err != nil { if errors.Is(err, nats.ErrNoResponders) { From 38ba3fec636e684f81320e5d8ab497f5f1e985a0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 13:00:34 +0000 Subject: [PATCH 20/75] fix(distributed): stop reclaiming healthy reconciler-driven loads The abandoned-load sweeper treated a replica row with no load job as abandoned. Only the request path creates load jobs; the reconciler's own scale-up loads a replica without one. So any scale-up that ran past the five-minute grace period was deleted mid-transfer, which for a multi-gigabyte checkpoint is every time. The replica never finished anywhere, and the reconciler kept re-placing it, so it looked like one replica hopping between nodes instead of a model reaching its replica count. A row with no job is now reclaimed only once its node stops being healthy, which is the case the sweeper was written for: a worker that dropped out mid-transfer. A job that failed or stopped heartbeating still proves abandonment on its own. Every uncertain case leaves the slot held. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- .../nodes/reconciler_abandoned_load.go | 69 +++++++++++-------- .../nodes/reconciler_abandoned_load_test.go | 17 ++++- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/core/services/nodes/reconciler_abandoned_load.go b/core/services/nodes/reconciler_abandoned_load.go index 9b6b7b654..f8d5dc702 100644 --- a/core/services/nodes/reconciler_abandoned_load.go +++ b/core/services/nodes/reconciler_abandoned_load.go @@ -35,11 +35,17 @@ var preServingStates = []string{"loading", "staging"} // there until an operator intervened: scheduling saw no free slot, and eviction // found nothing it was allowed to evict. // -// A row is abandoned when no live load job vouches for it. Ownership is decided -// by the job's LastProgress heartbeat rather than elapsed time, because staging -// a large checkpoint legitimately runs for a long while without touching the -// replica row. That is the same signal job takeover already trusts, so a -// transfer this sweeper reclaims is one no replica is still driving. +// A row is only reclaimed when something proves the load is not progressing: +// either a load job that has failed or stopped heartbeating, or, for a row with +// no job at all, a node that is no longer healthy. +// +// The no-job case has to be conservative. Only the request path creates load +// jobs; the reconciler's own scale-up loads a replica without one. Treating a +// missing job as proof of abandonment would let this sweeper delete a healthy +// reconciler-driven transfer the moment it ran past the grace period, which for +// a multi-gigabyte checkpoint is every time. A healthy node with no job is +// therefore left alone; when the node is gone, nothing can be progressing and +// the row is safe to reclaim. func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { if rc.db == nil { return @@ -56,7 +62,7 @@ func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { now := time.Now() for _, row := range stuck { - if rc.loadStillRunning(ctx, row.ModelName, now) { + if !rc.loadAbandoned(ctx, row, now) { continue } if err := rc.registry.RemoveNodeModel(ctx, row.NodeID, row.ModelName, row.ReplicaIndex); err != nil { @@ -70,29 +76,38 @@ func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) { } } -// loadStillRunning reports whether a load job is actively driving this model. +// loadAbandoned reports whether this row's load has demonstrably stopped. // -// A missing job means nobody is loading it. A failed job has already given up. -// An orphaned job stopped heartbeating, which is the condition another replica -// uses to take it over, so the transfer behind it is not progressing either. -// Any error reading the job is treated as "still running": leaving a slot held -// for one more pass costs a scheduling opportunity, while removing a row out -// from under a live transfer would restart a multi-gigabyte load. -func (rc *ReplicaReconciler) loadStillRunning(ctx context.Context, modelName string, now time.Time) bool { - job, err := rc.registry.GetLoadJob(ctx, modelName) - if errors.Is(err, gorm.ErrRecordNotFound) { - return false - } - if err != nil { +// Every uncertain case answers false. Leaving a slot held for another pass +// costs one scheduling opportunity; reclaiming a row out from under a live +// transfer restarts a multi-gigabyte load and, on a single-slot node, makes the +// model unschedulable there for as long as the retry loop runs. +func (rc *ReplicaReconciler) loadAbandoned(ctx context.Context, row NodeModel, now time.Time) bool { + job, err := rc.registry.GetLoadJob(ctx, row.ModelName) + switch { + case errors.Is(err, gorm.ErrRecordNotFound), err == nil && job == nil: + // No job: only the request path creates them, so this may be a healthy + // reconciler-driven load. Reclaim only once its node is gone. + return !rc.nodeHealthy(ctx, row.NodeID) + case err != nil: xlog.Warn("Reconciler: cannot read load job, leaving the replica slot held", - "model", modelName, "error", err) + "model", row.ModelName, "error", err) + return false + case job.State == LoadJobStateFailed: + return true + default: + return job.IsOrphaned(now) + } +} + +// nodeHealthy reports whether the row's node is still healthy. An unreadable +// node counts as healthy so a database blip cannot trigger a reclaim. +func (rc *ReplicaReconciler) nodeHealthy(ctx context.Context, nodeID string) bool { + node, err := rc.registry.Get(ctx, nodeID) + if err != nil || node == nil { + xlog.Warn("Reconciler: cannot read node for a stuck replica, leaving the slot held", + "node", nodeID, "error", err) return true } - if job == nil { - return false - } - if job.State == LoadJobStateFailed { - return false - } - return !job.IsOrphaned(now) + return node.Status == StatusHealthy } diff --git a/core/services/nodes/reconciler_abandoned_load_test.go b/core/services/nodes/reconciler_abandoned_load_test.go index 6740b13aa..3289fc31b 100644 --- a/core/services/nodes/reconciler_abandoned_load_test.go +++ b/core/services/nodes/reconciler_abandoned_load_test.go @@ -82,14 +82,29 @@ var _ = Describe("ReplicaReconciler — abandoned load sweeper", func() { Expect(rowExists("abandoned")).To(BeFalse()) }) - It("reclaims a loading row that has no load job at all", func() { + It("reclaims a jobless row once its node is gone", func() { seedReplica("orphan", "loading", time.Hour) + Expect(registry.MarkUnhealthy(context.Background(), node.ID)).To(Succeed()) rc.reclaimAbandonedLoads(context.Background()) Expect(rowExists("orphan")).To(BeFalse()) }) + // Only the request path creates load jobs. The reconciler's own scale-up + // loads a replica without one, so treating a missing job as abandonment + // deleted healthy transfers the moment they outran the grace period, which + // for a multi-gigabyte checkpoint is every time. That is what made a replica + // appear to hop between nodes instead of finishing anywhere. + It("keeps a jobless row while its node is still healthy", func() { + seedReplica("scaling-up", "staging", time.Hour) + + rc.reclaimAbandonedLoads(context.Background()) + + Expect(rowExists("scaling-up")).To(BeTrue(), + "a reconciler-driven load has no job row and must not be reclaimed for it") + }) + It("keeps a long transfer whose job is still heartbeating", func() { // The row itself is old, because staging does not touch it. Only the // job proves the transfer is alive. From 2c68fa1eb6b9e172711608076e5880470dc415eb Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 13:18:35 +0000 Subject: [PATCH 21/75] fix(distributed): keep eviction inside the model's node selector When no node the selector allows has a free slot, scheduling falls back to evicting the least-recently-used idle model. That eviction searched every healthy node, so it freed a slot on a node the selector forbids and the model was then placed there: pinned to one class of hardware and running on another. An unrelated model pays for it. On this cluster an embedding model pinned to Apple hardware could not reach its only matching node, so each attempt evicted a large language model from an Nvidia node, failed to start there anyway, and left the evicted model to reload. Repeated, that reads as one replica bouncing between nodes. Eviction is now restricted to the candidate set the selector produced. With no selector the candidate set is nil and eviction stays global. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/router.go | 28 ++++- .../nodes/router_eviction_selector_test.go | 110 ++++++++++++++++++ 2 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 core/services/nodes/router_eviction_selector_test.go diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 14b19092e..08cf2fd78 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1129,7 +1129,7 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // 4. Preemptive eviction: if no suitable node found, evict the LRU model with zero in-flight if node == nil { - evictedNode, evictErr := r.evictLRUAndFreeNode(ctx) + evictedNode, evictErr := r.evictLRUAndFreeNodeFrom(ctx, candidateNodeIDs) if evictErr != nil { if errors.Is(evictErr, ErrEvictionBusy) { return nil, "", 0, fmt.Errorf("no healthy nodes available: %w", evictErr) @@ -1153,7 +1153,7 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // it can race with another concurrent scheduler. xlog.Warn("Chosen node has no free replica slot, evicting LRU", "node", node.Name, "model", modelID, "max_slots", maxSlots) - evictedNode, evictErr := r.evictLRUAndFreeNode(ctx) + evictedNode, evictErr := r.evictLRUAndFreeNodeFrom(ctx, candidateNodeIDs) if evictErr != nil { return nil, "", 0, fmt.Errorf("no replica slot on %s and eviction failed: %w", node.Name, evictErr) } @@ -1979,7 +1979,23 @@ var ErrEvictionBusy = errors.New("all models busy, cannot evict") // Uses SELECT FOR UPDATE inside a transaction to prevent two frontends from // simultaneously picking the same eviction target. The NodeModel row is deleted // inside the transaction; the NATS unload command is sent after commit. +// evictLRUAndFreeNode evicts across every healthy node. Callers that hold a +// candidate set must use evictLRUAndFreeNodeFrom instead. func (r *SmartRouter) evictLRUAndFreeNode(ctx context.Context) (*BackendNode, error) { + return r.evictLRUAndFreeNodeFrom(ctx, nil) +} + +// evictLRUAndFreeNodeFrom evicts the least-recently-used idle model from one of +// candidateNodeIDs, or from any healthy node when the set is nil. +// +// Restricting eviction to the candidate set matters whenever the model being +// scheduled has a node selector. Evicting globally freed a slot on a node the +// selector forbids, so the model was then placed there anyway, on hardware it +// was explicitly pinned away from, and an unrelated model was dropped to make +// the room. On a cluster where the selector-matching node was momentarily +// unavailable this repeated, and the evicted model appeared to bounce between +// nodes. +func (r *SmartRouter) evictLRUAndFreeNodeFrom(ctx context.Context, candidateNodeIDs []string) (*BackendNode, error) { const maxEvictionRetries = 5 const evictionRetryInterval = 500 * time.Millisecond @@ -1991,7 +2007,7 @@ func (r *SmartRouter) evictLRUAndFreeNode(ctx context.Context) (*BackendNode, er var lru NodeModel err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // Lock the row so no other frontend can evict the same model - if err := currentModelRevision(tx.Clauses(clause.Locking{Strength: "UPDATE"})). + q := currentModelRevision(tx.Clauses(clause.Locking{Strength: "UPDATE"})). Joins("JOIN backend_nodes ON backend_nodes.id = node_models.node_id"). Where(`node_models.in_flight = 0 AND node_models.state = ? AND backend_nodes.status = ? AND ( @@ -2000,7 +2016,11 @@ func (r *SmartRouter) evictLRUAndFreeNode(ctx context.Context) (*BackendNode, er AND (NOT EXISTS (SELECT 1 FROM model_config_states mcs2 WHERE mcs2.model_name = nm2.model_name) OR nm2.config_revision = (SELECT mcs3.config_revision FROM model_config_states mcs3 WHERE mcs3.model_name = nm2.model_name))) > COALESCE((SELECT sc2.min_replicas FROM model_scheduling_configs sc2 WHERE sc2.model_name = node_models.model_name), 1) - )`, "loaded", StatusHealthy). + )`, "loaded", StatusHealthy) + if len(candidateNodeIDs) > 0 { + q = q.Where("node_models.node_id IN ?", candidateNodeIDs) + } + if err := q. Order("node_models.last_used ASC"). First(&lru).Error; err != nil { return err diff --git a/core/services/nodes/router_eviction_selector_test.go b/core/services/nodes/router_eviction_selector_test.go new file mode 100644 index 000000000..8d0caaffb --- /dev/null +++ b/core/services/nodes/router_eviction_selector_test.go @@ -0,0 +1,110 @@ +package nodes + +import ( + "context" + "fmt" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/testutil" +) + +// When no node the selector allows has a free slot, scheduling falls back to +// evicting the globally least-recently-used model. That eviction knew nothing +// about the selector, so a model pinned to one class of hardware would evict an +// unrelated model from a node it is not allowed to run on, and then be placed +// there. Two models lose: the pinned one runs on the wrong hardware, and the +// evicted one is dropped for nothing and has to reload elsewhere. +var _ = Describe("Eviction under a node selector", func() { + var ( + db *gorm.DB + registry *NodeRegistry + router *SmartRouter + ctx context.Context + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + router = NewSmartRouter(registry, SmartRouterOptions{DB: db}) + ctx = context.Background() + }) + + register := func(name string) *BackendNode { + node := &BackendNode{Name: name, NodeType: NodeTypeBackend, Address: name + ":50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + fetched, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + return fetched + } + + rowID := 0 + seed := func(node *BackendNode, model string, idleFor time.Duration, inFlight int) { + rowID++ + Expect(db.Create(&NodeModel{ + ID: fmt.Sprintf("row-%d", rowID), NodeID: node.ID, ModelName: model, + Address: node.Address, State: "loaded", InFlight: inFlight, + LastUsed: time.Now().Add(-idleFor), UpdatedAt: time.Now(), + }).Error).To(Succeed()) + } + seedLoaded := func(node *BackendNode, model string, idleFor time.Duration) { + seed(node, model, idleFor, 0) + } + + rowExists := func(model string) bool { + var n int64 + Expect(db.Model(&NodeModel{}).Where("model_name = ?", model).Count(&n).Error).To(Succeed()) + return n > 0 + } + + It("does not evict from a node the selector excludes", func() { + allowed := register("allowed-node") + excluded := register("excluded-node") + // The only eviction candidate sits on the excluded node and is the + // global LRU, so an unconstrained eviction would take it. + seedLoaded(excluded, "innocent-bystander", time.Hour) + // In-flight, so it is not an eviction candidate: the allowed node has + // nothing that can be freed. + seed(allowed, "busy-here", time.Minute, 1) + + _, err := router.evictLRUAndFreeNodeFrom(ctx, []string{allowed.ID}) + + Expect(err).To(HaveOccurred(), "no eviction candidate exists on an allowed node") + Expect(rowExists("innocent-bystander")).To(BeTrue(), + "a model on a node the selector excludes must not be evicted to make room") + }) + + It("evicts the LRU among the allowed nodes only", func() { + allowed := register("allowed-node") + excluded := register("excluded-node") + seedLoaded(excluded, "older-elsewhere", 2*time.Hour) + seedLoaded(allowed, "newer-but-allowed", time.Hour) + + node, err := router.evictLRUAndFreeNodeFrom(ctx, []string{allowed.ID}) + + Expect(err).ToNot(HaveOccurred()) + Expect(node.ID).To(Equal(allowed.ID)) + Expect(rowExists("newer-but-allowed")).To(BeFalse()) + Expect(rowExists("older-elsewhere")).To(BeTrue()) + }) + + It("keeps evicting globally when the model has no selector", func() { + a := register("node-a") + seedLoaded(a, "anything", time.Hour) + + node, err := router.evictLRUAndFreeNodeFrom(ctx, nil) + + Expect(err).ToNot(HaveOccurred()) + Expect(node.ID).To(Equal(a.ID)) + Expect(rowExists("anything")).To(BeFalse()) + }) +}) From 1dc3aeef8773c128d708a58eea3c72c687e1dee2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 19:11:16 +0000 Subject: [PATCH 22/75] fix(distributed): resolve config revisions through one entry point A model's revision is published by administration and checked against on every inference request. Those were computed by separate code: the request path resolves through the loader, while each publisher hashed whatever ModelConfig it happened to hold. By then SetDefaults had folded in the GGUF guess and app-level options, so the published value was one no request would ever carry and the model became unroutable until the row was deleted by hand. Fixing the publishers one at a time did not hold. Three rounds each found another: the startup resync, then a saved edit and a toggle, then a rename and the peer-change path. ModelConfigLoader.RevisionFor is now the only way to obtain a revision, and the raw hash is unexported, so a caller outside this package cannot hash a config it holds. A publisher and a request agree by construction rather than by two implementations happening to match. The request path no longer falls back to hashing its merged config either: an unstamped config is routed without a revision rather than with a wrong one. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/backend/options.go | 18 ++-- core/config/model_config.go | 2 +- core/config/model_config_loader.go | 35 ++++++ core/config/model_config_revision.go | 11 +- .../model_config_revision_stability_test.go | 5 +- core/config/model_config_revision_test.go | 7 +- .../http/endpoints/localai/edit_model_test.go | 8 +- core/services/modeladmin/config.go | 15 +-- core/services/modeladmin/remote_sync.go | 16 +-- core/services/modeladmin/remote_sync_test.go | 14 +-- .../modeladmin/revision_agreement_test.go | 102 ++++++++++++++++++ core/services/modeladmin/revision_resync.go | 8 +- core/services/modeladmin/state.go | 8 +- .../nodes/router_revision_lifecycle_test.go | 4 +- 14 files changed, 189 insertions(+), 64 deletions(-) create mode 100644 core/services/modeladmin/revision_agreement_test.go diff --git a/core/backend/options.go b/core/backend/options.go index 4f7c81483..93a6eadc0 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -202,18 +202,18 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo model.WithContext(so.Context), model.WithModelID(c.ModelID()), } - // Prefer the revision stamped when the configuration was loaded. c has since - // been merged with this request's prediction parameters (temperature, top_p, - // stop, ...), and hashing it here would produce a different revision for - // every distinct request body — which the controller reads as a config - // change and rejects as stale. Recomputing is the fallback for a config that - // never passed through the loader. + // Use the revision stamped when the configuration was parsed, and only + // that. By this point c has been merged with the request's prediction + // parameters and had SetDefaults applied, so hashing it here would produce + // a revision that depends on the request body and on whether the model file + // parsed, which the controller reads as a config change and rejects. Every + // config the loader hands out is stamped; an unstamped one was synthesized + // elsewhere and is routed without a revision rather than with a wrong one. if revision := c.PersistedConfigRevision(); revision != "" { defOpts = append(defOpts, model.WithConfigRevision(revision)) - } else if revision, err := config.ModelConfigRevision(&c); err == nil { - defOpts = append(defOpts, model.WithConfigRevision(revision)) } else { - xlog.Warn("Failed to compute model configuration revision", "model", c.ModelID(), "error", err) + xlog.Warn("Model configuration carries no revision stamp; routing without one", + "model", c.ModelID()) } managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil if managedPrimary { diff --git a/core/config/model_config.go b/core/config/model_config.go index c6121eb8c..600519c7f 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1864,7 +1864,7 @@ func (c *ModelConfig) PersistedConfigRevision() string { // persisted. It is computed from the receiver as-is, so callers must invoke it // only on a configuration that has not been merged with request overrides. func (c *ModelConfig) StampPersistedConfigRevision() error { - revision, err := ModelConfigRevision(c) + revision, err := modelConfigRevision(c) if err != nil { return err } diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go index 2a062c39d..b91449ff0 100644 --- a/core/config/model_config_loader.go +++ b/core/config/model_config_loader.go @@ -965,3 +965,38 @@ func hasAnyMappingKey(mapping *yaml.Node, keys ...string) bool { func nonemptyScalar(node *yaml.Node) bool { return node != nil && node.Kind == yaml.ScalarNode && node.Tag == "!!str" && strings.TrimSpace(node.Value) != "" } + +// RevisionFor returns the config revision for modelName: the one an inference +// request for that model will carry. +// +// This is the only way to obtain a revision outside this package. Every +// publisher must use it, so that what is published and what is checked are +// the same value by construction rather than by two implementations happening +// to agree. Hashing a ModelConfig directly is not available to callers, because +// a config that has been through SetDefaults or the request middleware hashes +// to something no request will ever present. +func (bcl *ModelConfigLoader) RevisionFor(modelName string, appConfig *ApplicationConfig) (string, error) { + cfg, err := bcl.LoadModelConfigFileByNameDefaultOptions(modelName, appConfig) + if err != nil { + return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err) + } + return stampedRevision(cfg, modelName) +} + +// RevisionForPath is RevisionFor for callers that hold loader options and a +// models path rather than an ApplicationConfig. +func (bcl *ModelConfigLoader) RevisionForPath(modelName, modelPath string, opts ...ConfigLoaderOption) (string, error) { + cfg, err := bcl.LoadModelConfigFileByName(modelName, modelPath, opts...) + if err != nil { + return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err) + } + return stampedRevision(cfg, modelName) +} + +func stampedRevision(cfg *ModelConfig, modelName string) (string, error) { + revision := cfg.PersistedConfigRevision() + if revision == "" { + return "", fmt.Errorf("no config revision stamped for %q", modelName) + } + return revision, nil +} diff --git a/core/config/model_config_revision.go b/core/config/model_config_revision.go index e8dd1bee5..8cbad2d6b 100644 --- a/core/config/model_config_revision.go +++ b/core/config/model_config_revision.go @@ -10,10 +10,17 @@ import ( "google.golang.org/protobuf/proto" ) -// ModelConfigRevision returns a stable revision of the persisted semantic +// modelConfigRevision returns a stable revision of the persisted semantic // configuration. ModelConfig's JSON tags exclude runtime-derived state and // source bookkeeping, while encoding/json orders map keys deterministically. -func ModelConfigRevision(cfg *ModelConfig) (string, error) { +// +// Deliberately unexported. It must only ever be called on a configuration as +// parsed from disk, before SetDefaults folds in the GGUF guess, the hardware +// defaults and app-level options. Callers outside this package cannot tell +// which they hold, and every time one hashed a defaulted or request-merged +// config it published a revision no inference request would carry, which makes +// the model unroutable. Use ModelConfigLoader.RevisionFor instead. +func modelConfigRevision(cfg *ModelConfig) (string, error) { if cfg == nil { return "", errors.New("model config is nil") } diff --git a/core/config/model_config_revision_stability_test.go b/core/config/model_config_revision_stability_test.go index b353f3d14..433b8e401 100644 --- a/core/config/model_config_revision_stability_test.go +++ b/core/config/model_config_revision_stability_test.go @@ -51,9 +51,8 @@ template: Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) cfg, ok := loader.GetModelConfig("example") Expect(ok).To(BeTrue()) - revision, err := config.ModelConfigRevision(&cfg) - Expect(err).ToNot(HaveOccurred()) - return revision + Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty()) + return cfg.PersistedConfigRevision() } It("does not change when the same file is loaded repeatedly", func() { diff --git a/core/config/model_config_revision_test.go b/core/config/model_config_revision_test.go index 0b95d3549..b75937271 100644 --- a/core/config/model_config_revision_test.go +++ b/core/config/model_config_revision_test.go @@ -19,10 +19,11 @@ var _ = Describe("Model configuration revisions", func() { return cfg } + // The raw hash is unexported on purpose, so these specs exercise it the way + // every caller now must: by stamping the parsed config. revision := func(cfg *config.ModelConfig) string { - value, err := config.ModelConfigRevision(cfg) - Expect(err).NotTo(HaveOccurred()) - return value + Expect(cfg.StampPersistedConfigRevision()).To(Succeed()) + return cfg.PersistedConfigRevision() } It("is stable across equivalent YAML formatting and map order", func() { diff --git a/core/http/endpoints/localai/edit_model_test.go b/core/http/endpoints/localai/edit_model_test.go index 17f7c4a7c..223943e46 100644 --- a/core/http/endpoints/localai/edit_model_test.go +++ b/core/http/endpoints/localai/edit_model_test.go @@ -294,9 +294,9 @@ var _ = Describe("Edit Model test", func() { Expect(client.published[0]).To(Equal(messaging.CacheInvalidateEvent{ Element: "old", Op: "delete", ConfigRevision: modeladmin.DeletedModelConfigRevision("old"), })) - newConfig, ok := loader.GetModelConfig("new") + _, ok := loader.GetModelConfig("new") Expect(ok).To(BeTrue()) - newRevision, err := config.ModelConfigRevision(&newConfig) + newRevision, err := loader.RevisionForPath("new", tempDir) Expect(err).ToNot(HaveOccurred()) Expect(client.published[1]).To(Equal(messaging.CacheInvalidateEvent{ Element: "new", Op: "install", ConfigRevision: newRevision, @@ -313,9 +313,9 @@ var _ = Describe("Edit Model test", func() { } _, oldOnPeer := peerLoader.GetModelConfig("old") Expect(oldOnPeer).To(BeFalse()) - peerConfig, newOnPeer := peerLoader.GetModelConfig("new") + _, newOnPeer := peerLoader.GetModelConfig("new") Expect(newOnPeer).To(BeTrue()) - peerRevision, err := config.ModelConfigRevision(&peerConfig) + peerRevision, err := peerLoader.RevisionForPath("new", tempDir) Expect(err).ToNot(HaveOccurred()) Expect(peerRevision).To(Equal(newRevision)) Expect(peerLifecycle.batches).To(Equal([][]modeladmin.ModelRevisionTransition{ diff --git a/core/services/modeladmin/config.go b/core/services/modeladmin/config.go index 515d014f7..2cadfcaed 100644 --- a/core/services/modeladmin/config.go +++ b/core/services/modeladmin/config.go @@ -185,13 +185,9 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[ // because SetDefaults runs again on the request path and is not // idempotent for every model, and the edit would leave the model // unroutable. - resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(updated.Name, s.AppConfig) + revision, err := s.Loader.RevisionFor(updated.Name, s.AppConfig) if err != nil { - return fmt.Errorf("resolve config revision: %w", err) - } - revision := resolved.PersistedConfigRevision() - if revision == "" { - return fmt.Errorf("no config revision stamped for %q", updated.Name) + return err } _ = s.Loader.Preload(s.modelsPath()) pending, err := s.applyRevision(ctx, name, updated.Name, revision, updated.IsDisabled()) @@ -351,13 +347,12 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte) if err := s.Loader.LoadModelConfigsFromPath(modelsPath, s.AppConfig.ToConfigLoaderOptions()...); err != nil { return fmt.Errorf("reload configs: %w", err) } - loaded, ok := s.Loader.GetModelConfig(req.Name) - if !ok { + if _, ok := s.Loader.GetModelConfig(req.Name); !ok { return fmt.Errorf("reload configs: model %q missing", req.Name) } - revision, err := config.ModelConfigRevision(&loaded) + revision, err := s.Loader.RevisionFor(req.Name, s.AppConfig) if err != nil { - return fmt.Errorf("compute config revision: %w", err) + return err } if err := s.Loader.Preload(modelsPath); err != nil { return fmt.Errorf("preload after edit: %w", err) diff --git a/core/services/modeladmin/remote_sync.go b/core/services/modeladmin/remote_sync.go index 844239a88..9eec4fa5e 100644 --- a/core/services/modeladmin/remote_sync.go +++ b/core/services/modeladmin/remote_sync.go @@ -49,9 +49,9 @@ func applyRemoteChange(ctx context.Context, cl *config.ModelConfigLoader, models disabled := true if exists { var err error - revision, err = config.ModelConfigRevision(&cfg) + revision, err = authoritative.RevisionForPath(name, modelsPath, opts...) if err != nil { - return fmt.Errorf("compute authoritative model config revision for %q: %w", name, err) + return fmt.Errorf("resolve authoritative model config revision for %q: %w", name, err) } disabled = cfg.IsDisabled() } @@ -83,15 +83,9 @@ func changedConfigNames(current, snapshot map[string]config.ModelConfig, named s changed[name] = struct{}{} continue } - previousRevision, err := config.ModelConfigRevision(&previous) - if err != nil { - return nil, fmt.Errorf("compute current model config revision for %q: %w", name, err) - } - revision, err := config.ModelConfigRevision(&cfg) - if err != nil { - return nil, fmt.Errorf("compute authoritative model config revision for %q: %w", name, err) - } - if previousRevision != revision { + // Both sides come from a loader, so both carry the revision stamped + // when their file was parsed. Comparing the stamps compares the files. + if previous.PersistedConfigRevision() != cfg.PersistedConfigRevision() { changed[name] = struct{}{} } } diff --git a/core/services/modeladmin/remote_sync_test.go b/core/services/modeladmin/remote_sync_test.go index 32289429f..d5a2b4664 100644 --- a/core/services/modeladmin/remote_sync_test.go +++ b/core/services/modeladmin/remote_sync_test.go @@ -58,9 +58,9 @@ var _ = Describe("ApplyRemoteChange", func() { Expect(ApplyRemoteChange(context.Background(), loader, dir, evt, lifecycle)).To(Succeed()) Expect(ApplyRemoteChange(context.Background(), loader, dir, evt, lifecycle)).To(Succeed()) Expect(lifecycle.calls).To(HaveLen(2)) - loaded, ok := loader.GetModelConfig("peer-alias") + _, ok := loader.GetModelConfig("peer-alias") Expect(ok).To(BeTrue()) - revision, err := config.ModelConfigRevision(&loaded) + revision, err := loader.RevisionForPath("peer-alias", dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.calls[0].revision).To(Equal(revision)) Expect(lifecycle.calls[1].revision).To(Equal(revision)) @@ -84,7 +84,7 @@ var _ = Describe("ApplyRemoteChange", func() { loaded, ok := loader.GetModelConfig("peer-alias") Expect(ok).To(BeTrue()) Expect(loaded.ContextSize).To(HaveValue(Equal(10000))) - revision, err := config.ModelConfigRevision(&loaded) + revision, err := loader.RevisionForPath(loaded.Name, dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.calls).To(HaveLen(3)) Expect(lifecycle.calls[1].revision).To(Equal(revision)) @@ -100,7 +100,7 @@ var _ = Describe("ApplyRemoteChange", func() { loaded, ok := loader.GetModelConfig("reinstalled") Expect(ok).To(BeTrue()) - revision, err := config.ModelConfigRevision(&loaded) + revision, err := loader.RevisionForPath(loaded.Name, dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.calls).To(HaveLen(1)) Expect(lifecycle.calls[0].revision).To(Equal(revision)) @@ -172,7 +172,7 @@ var _ = Describe("ApplyRemoteChange", func() { Expect(loaded.ContextSize).To(HaveValue(Equal(10000))) _, ok = loader.GetModelConfig("deleted") Expect(ok).To(BeFalse()) - changedRevision, err := config.ModelConfigRevision(&loaded) + changedRevision, err := loader.RevisionForPath(loaded.Name, dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.calls).To(ConsistOf( revisionLifecycleCall{oldName: "changed", newName: "changed", revision: changedRevision}, @@ -228,7 +228,7 @@ var _ = Describe("ApplyRemoteChange", func() { loaded, ok := loader.GetModelConfig("ordered") Expect(ok).To(BeTrue()) Expect(loaded.ContextSize).To(HaveValue(Equal(10000))) - revision, err := config.ModelConfigRevision(&loaded) + revision, err := loader.RevisionForPath(loaded.Name, dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.revisions()).To(HaveLen(2)) Expect(lifecycle.revisions()[1]).To(Equal(revision)) @@ -263,7 +263,7 @@ var _ = Describe("ApplyRemoteChange", func() { Expect(ok).To(BeTrue()) Expect(loaded.ContextSize).To(HaveValue(Equal(10000))) Expect(readMap(filepath.Join(dir, "ordered.yaml"))).To(HaveKeyWithValue("context_size", 10000)) - revision, err := config.ModelConfigRevision(&loaded) + revision, err := loader.RevisionForPath(loaded.Name, dir) Expect(err).ToNot(HaveOccurred()) Expect(lifecycle.revisions()).To(HaveLen(2)) Expect(lifecycle.revisions()[1]).To(Equal(revision)) diff --git a/core/services/modeladmin/revision_agreement_test.go b/core/services/modeladmin/revision_agreement_test.go new file mode 100644 index 000000000..fcc9616b9 --- /dev/null +++ b/core/services/modeladmin/revision_agreement_test.go @@ -0,0 +1,102 @@ +package modeladmin + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/system" +) + +// A model's revision is published by administration and checked against on +// every inference request. Those were computed by different code, and each time +// they drifted the model became unroutable until someone deleted the row by +// hand: the request path resolves through the loader, while publishers hashed +// whatever ModelConfig they were holding, which by then had SetDefaults applied. +// +// There is now one resolver, ModelConfigLoader.RevisionFor, and the raw hash is +// unexported so a new publisher cannot reintroduce the split. This pins the +// property that mattered: whatever a publisher writes is what a request brings. +var _ = Describe("Published and requested revisions agree", func() { + var ( + dir string + appConfig *config.ApplicationConfig + loader *config.ModelConfigLoader + ) + + // Several shapes, because the divergence only ever showed up on configs + // rich enough for SetDefaults to change something: a model file to guess + // from, several derived usecases, explicit options. + models := map[string]string{ + "plain": "name: plain\nbackend: llama-cpp\nparameters:\n model: plain.gguf\n", + "multimodal": "name: multimodal\nbackend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\nmmproj: mm/mmproj.gguf\noptions:\n - use_jinja:true\n - parallel:2\nparameters:\n model: mm/model.gguf\n", + "auto-ctx": "name: auto-ctx\nbackend: llama-cpp\ncontext_size: -1\nparameters:\n model: auto.gguf\n", + "no-backend": "name: no-backend\nparameters:\n model: bare.gguf\n", + "with-thread": "name: with-thread\nbackend: llama-cpp\nthreads: 3\nparameters:\n model: t.gguf\n", + } + + BeforeEach(func() { + dir = GinkgoT().TempDir() + for name, body := range models { + Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed()) + } + appConfig = config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + appConfig.Threads = 8 + loader = config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + }) + + // requestRevision mirrors what core/backend.ModelOptions forwards to the + // router: the stamp on the config the request pipeline resolved. + requestRevision := func(name string) string { + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig) + Expect(err).ToNot(HaveOccurred()) + return cfg.PersistedConfigRevision() + } + + It("resolves the same revision a request will carry, for every model shape", func() { + for name := range models { + published, err := loader.RevisionFor(name, appConfig) + Expect(err).ToNot(HaveOccurred(), "model %s", name) + Expect(published).To(Equal(requestRevision(name)), "model %s: publisher and request disagree", name) + } + }) + + It("resolves the same revision through the path-based form", func() { + for name := range models { + byAppConfig, err := loader.RevisionFor(name, appConfig) + Expect(err).ToNot(HaveOccurred()) + byPath, err := loader.RevisionForPath(name, dir, appConfig.ToConfigLoaderOptions()...) + Expect(err).ToNot(HaveOccurred()) + Expect(byPath).To(Equal(byAppConfig), "model %s", name) + } + }) + + It("does not move when the app-level defaults change", func() { + before := map[string]string{} + for name := range models { + r, err := loader.RevisionFor(name, appConfig) + Expect(err).ToNot(HaveOccurred()) + before[name] = r + } + + other := config.NewApplicationConfig() + other.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + other.Threads = 1 + other.F16 = true + other.ContextSize = 4096 + fresh := config.NewModelConfigLoader(dir) + Expect(fresh.LoadModelConfigsFromPath(dir, other.ToConfigLoaderOptions()...)).To(Succeed()) + + for name := range models { + r, err := fresh.RevisionFor(name, other) + Expect(err).ToNot(HaveOccurred()) + Expect(r).To(Equal(before[name]), + "model %s: changing an app-level setting must not make every model unroutable", name) + } + }) +}) diff --git a/core/services/modeladmin/revision_resync.go b/core/services/modeladmin/revision_resync.go index 5d84831a2..7d92f52a4 100644 --- a/core/services/modeladmin/revision_resync.go +++ b/core/services/modeladmin/revision_resync.go @@ -87,13 +87,9 @@ func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigL // (it re-runs the GGUF guess and hardware defaults), so hashing the // stored config yields a value no request will ever carry, and // publishing it would wedge the model this resync exists to unwedge. - resolved, err := loader.LoadModelConfigFileByNameDefaultOptions(cfg.Name, appConfig) + want, err := loader.RevisionFor(cfg.Name, appConfig) if err != nil { - return fmt.Errorf("resolve config for %q: %w", cfg.Name, err) - } - want := resolved.PersistedConfigRevision() - if want == "" { - return fmt.Errorf("no config revision stamped for %q", cfg.Name) + return err } stored, err := store.GetModelConfigRevision(ctx, cfg.Name) diff --git a/core/services/modeladmin/state.go b/core/services/modeladmin/state.go index 4f84b5859..d37368d9d 100644 --- a/core/services/modeladmin/state.go +++ b/core/services/modeladmin/state.go @@ -68,13 +68,9 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act // because SetDefaults runs again on the request path and is not // idempotent for every model, and the edit would leave the model // unroutable. - resolved, err := s.Loader.LoadModelConfigFileByNameDefaultOptions(name, s.AppConfig) + revision, err := s.Loader.RevisionFor(name, s.AppConfig) if err != nil { - return fmt.Errorf("resolve config revision: %w", err) - } - revision := resolved.PersistedConfigRevision() - if revision == "" { - return fmt.Errorf("no config revision stamped for %q", name) + return err } pending, err := s.applyRevision(ctx, name, name, revision, action == ActionDisable) if err != nil { diff --git a/core/services/nodes/router_revision_lifecycle_test.go b/core/services/nodes/router_revision_lifecycle_test.go index b9b78f790..7b1de6144 100644 --- a/core/services/nodes/router_revision_lifecycle_test.go +++ b/core/services/nodes/router_revision_lifecycle_test.go @@ -249,8 +249,8 @@ var _ = Describe("revision-bound load publication", func() { LLMConfig: config.LLMConfig{ContextSize: &contextSize}, } cfg.Model = "models/full-flow.gguf" - expectedRevision, err := config.ModelConfigRevision(&cfg) - Expect(err).NotTo(HaveOccurred()) + Expect(cfg.StampPersistedConfigRevision()).To(Succeed()) + expectedRevision := cfg.PersistedConfigRevision() router := NewSmartRouter(registry, SmartRouterOptions{ Unloader: unloader, From f7ded96b1ec50a2b3df49771fb20500a6c9fd11e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 24 Aug 2026 19:58:49 +0000 Subject: [PATCH 23/75] fix(distributed): probe liveness on a subject every worker answers The scheduler's liveness probe asks a worker a question over NATS and reads "no responders" as proof the worker is gone. That is only sound when every worker in the fleet subscribes to the subject asked. It asked models.running, which arrived in 4.6. A 4.5 worker is alive and serving, answers backend.list, and never subscribes to models.running, so the probe condemned it on every scheduling attempt and marked it unhealthy. A model pinned to such a node by its selector could then never be placed at all: on this cluster an embedding model pinned to the one Apple node was unschedulable for exactly this reason, while that node's log showed it handling backend.list throughout. Ask backend.list, which has been in the worker protocol far longer, and treat a worker that answers anything as alive. Only a node that reports no responders on every subject is absent, so adding a newer subject here can never condemn an older worker. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Code:claude-opus-5 [golangci-lint] --- core/services/nodes/unloader.go | 39 ++++++++++++----- core/services/nodes/unloader_ping_test.go | 51 +++++++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 core/services/nodes/unloader_ping_test.go diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 3b1cd15b8..460be8acf 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -372,17 +372,36 @@ func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendL // scheduler could pick a node that could not be given work and the request // failed with "no responders available". // -// It reuses the models.running subject rather than a dedicated ping subject on -// purpose: a new subject would go unanswered by any worker that has not been -// upgraded yet, and this check would then report every one of them as dead. -// The worker answers out of its in-memory process table, so a live node -// replies immediately, and NATS reports no-responders without waiting out the -// timeout. +// The subject asked has to be one every worker in the fleet subscribes to, or +// this check condemns the workers that do not. models.running was the obvious +// choice and the wrong one: it arrived in 4.6, so a 4.5 worker that is alive +// and serving never answers it, and a model pinned to that node could never be +// scheduled. backend.list has been part of the worker protocol far longer, so +// it is the safer question to ask. +// +// A worker that answers anything is alive. Only when every subject reports no +// responders is the node treated as absent, so adding a newer subject here can +// never condemn an older worker. func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { - subject := messaging.SubjectNodeModelsRunning(nodeID) - _, err := messaging.RequestJSON[messaging.ModelsRunningRequest, messaging.ModelsRunningReply]( - a.nats, subject, messaging.ModelsRunningRequest{}, 5*time.Second) - return err + subjects := []string{ + messaging.SubjectNodeBackendList(nodeID), + messaging.SubjectNodeModelsRunning(nodeID), + } + var lastErr error + for _, subject := range subjects { + _, err := messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply]( + a.nats, subject, messaging.BackendListRequest{}, 5*time.Second) + if err == nil { + return nil + } + if !errors.Is(err, nats.ErrNoResponders) { + // Reached someone, or failed for a reason that is not absence. + // Either way the node is not proven gone. + return nil + } + lastErr = err + } + return lastErr } // ListRunningModels asks a worker node which model backend processes it diff --git a/core/services/nodes/unloader_ping_test.go b/core/services/nodes/unloader_ping_test.go new file mode 100644 index 000000000..a9b3a5889 --- /dev/null +++ b/core/services/nodes/unloader_ping_test.go @@ -0,0 +1,51 @@ +package nodes + +import ( + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/nats-io/nats.go" + + "github.com/mudler/LocalAI/core/services/messaging" +) + +// The scheduler's liveness probe asks a worker a question over NATS and treats +// "no responders" as proof the worker is gone. That is only sound if every +// worker in the fleet subscribes to the subject asked. +// +// It originally asked models.running, which arrived in 4.6. A 4.5 worker is +// perfectly alive and serving, answers backend.list, and never subscribes to +// models.running, so the probe condemned it on every scheduling attempt. A +// model pinned to such a node could then never be placed at all. +var _ = Describe("Node liveness probe subject", func() { + var ( + mc *scriptedMessagingClient + adapter *RemoteUnloaderAdapter + ) + + const nodeID = "11111111-2222-3333-4444-555555555555" + + BeforeEach(func() { + mc = newScriptedMessagingClient() + adapter = NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + }) + + It("treats a worker that answers backend.list as alive", func() { + // A worker old enough to predate models.running: it answers the + // long-standing backend.list subject and nothing else. + mc.scriptReply(messaging.SubjectNodeBackendList(nodeID), messaging.BackendListReply{}) + mc.scriptNoResponders(messaging.SubjectNodeModelsRunning(nodeID)) + + Expect(errors.Is(adapter.PingNode(nodeID), nats.ErrNoResponders)).To(BeFalse(), + "a worker answering backend.list is alive regardless of newer subjects") + }) + + It("still reports a worker that answers nothing as absent", func() { + mc.scriptNoResponders(messaging.SubjectNodeBackendList(nodeID)) + mc.scriptNoResponders(messaging.SubjectNodeModelsRunning(nodeID)) + + Expect(errors.Is(adapter.PingNode(nodeID), nats.ErrNoResponders)).To(BeTrue()) + }) +}) From 496921f73a207636ffd57274e40a9ea6ac9b8c4e Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Mon, 24 Aug 2026 23:41:38 +0200 Subject: [PATCH 24/75] chore(model-gallery): :arrow_up: update checksum (#11707) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index c980a79fd..1c67d799b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -558,10 +558,10 @@ files: - filename: llama-cpp/models/ornith-1.5-9b/Ornith-1.5-9B-Q4_K_M.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/Ornith-1.5-9B-Q4_K_M.gguf - sha256: 7d791afcb31812acc88cd5aafc675391df28c6fc3d8eae002bb4e6cc3d8cfd8d + sha256: 70c112196e0b7023803c9762752e46d29e612a92c83f995bc3ba1ceb07e8fab6 - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf - sha256: d65001a94c4b6852bc7a0e7c5cc92fe8506755bb270e54483fd5feec7ae39a19 + sha256: 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d - !!merge <<: *ornith-1-5-9b name: "ornith-1.5-9b-q8" variants: [] @@ -593,10 +593,10 @@ files: - filename: llama-cpp/models/ornith-1.5-9b/Ornith-1.5-9B-Q8_0.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/Ornith-1.5-9B-Q8_0.gguf - sha256: 6874eeb25c71081dc8f0bbe88f3ebb786312447132745371cd980bce95d259b9 + sha256: 22086870b009dbe9815ee752c48a82de930118a7c5ce5599590892ae03b8b010 - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf - sha256: d65001a94c4b6852bc7a0e7c5cc92fe8506755bb270e54483fd5feec7ae39a19 + sha256: 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: @@ -654,7 +654,7 @@ files: - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q4_K_M.gguf - sha256: c5e4fe705883e244a468c9e445c8d6ba37fd310b0113e25d2b8a7f2d6f1243e8 + sha256: 1f74330b211a8253c96f1bf586cba6eb56d37117c97ed9e6eec18c198a4e7fe5 - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 @@ -687,7 +687,7 @@ files: - filename: llama-cpp/models/qwen3.8-27b-obliterated/Qwen3.8-27B-OBLITERATED-Q8_0.gguf uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/Qwen3.8-27B-OBLITERATED-Q8_0.gguf - sha256: 4ed72a101dfa7f8fd642598368c4d334f1334cedc5254b71a06b5c4a542c59fc + sha256: afa839b2fa5bc890e5735031dda2c6239d3b6bba3b6ffa29477cbc14a2e1f221 - filename: llama-cpp/mmproj/qwen3.8-27b-obliterated/mmproj-model-bf16.gguf uri: huggingface://OBLITERATUS/Qwen3.8-27B-OBLITERATED/mmproj-model-bf16.gguf sha256: e484e3b7e907ed0e0644c0de56c3f5929c7ad5c9c6cc84d35a9d8dc08d461545 From 964be3bceb4c15f09505a82abdb1695692fb4660 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:54:29 +0200 Subject: [PATCH 25/75] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `0ed847d3140baead542abe3e5e6fe841013e7340` (#11708) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index 847ddbb20..f3ff79a7f 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9 +IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From a760a7ab4bd16e3e6c41ba8f543a4c9188a2760e Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:52:57 +0800 Subject: [PATCH 26/75] fix(backends): honor enable_thinking=false in sglang and vllm (#11715) Those backends only forwarded the flag when it was "true", so "false" never reached apply_chat_template and Qwen3 kept thinking on. Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- backend/python/sglang/backend.py | 5 +++-- backend/python/sglang/test.py | 32 ++++++++++++++++++++++++++++++++ backend/python/vllm/backend.py | 6 +++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index c28b59a0e..ad6c6ca10 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -363,8 +363,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): template_kwargs["tools"] = json.loads(request.Tools) except json.JSONDecodeError: pass - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 92688f444..deb615883 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -96,6 +96,38 @@ class TestSglangHelpers(unittest.TestCase): servicer._apply_engine_args({}, "[1,2,3]") self.assertIn("must be a JSON object", str(ctx.exception)) + def test_build_prompt_forwards_enable_thinking(self): + from types import SimpleNamespace + + class Tok: + def __init__(self): + self.kwargs = None + + def apply_chat_template(self, messages, **kwargs): + self.kwargs = kwargs + return "PROMPT" + + def kwargs_for(metadata): + servicer = self._servicer() + tok = Tok() + servicer.tokenizer = tok + msg = SimpleNamespace( + role="user", content="hi", name="", + tool_call_id="", reasoning_content="", tool_calls="", + ) + req = SimpleNamespace( + Prompt="", UseTokenizerTemplate=True, + Messages=[msg], Tools="", Metadata=metadata, + ) + self.assertEqual(servicer._build_prompt(req), "PROMPT") + return tok.kwargs + + self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True) + # "false" used to be dropped, so Qwen3 kept thinking on + self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False) + self.assertNotIn("enable_thinking", kwargs_for({})) + self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + if __name__ == "__main__": unittest.main() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 8fd3c2dc1..f3f01ec45 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -587,9 +587,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): except json.JSONDecodeError: pass - # Enable thinking mode if requested - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) From f7c55788c7fdab3ba6de234fa3f8788f91bcd4ed Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 25 Aug 2026 12:54:00 +0200 Subject: [PATCH 27/75] feat(gallery): add Ornith 1.5 35B variants (#11714) Add the official Q4_K_M and Q8_0 GGUF builds with their shared BF16 vision projector. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 97 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 1c67d799b..35392e405 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -597,6 +597,103 @@ - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf sha256: 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d +- &ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q4" + variants: + - model: ornith-1.5-35b-a3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF + description: | + Ornith-1.5-35B-A3B is an MIT-licensed Qwen3.5 mixture-of-experts model + from Ornith AI for agentic coding, reasoning, repository-level software + tasks, and tool use. It activates about 3B parameters per token and + supports text and image input with a context window of 262K tokens. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. A + higher-quality Q8_0 model is available as a variant. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q4_K_M.gguf + sha256: 42739874cc2ccfdb8523b23fbe52e29b2a7555c8176737ca9ca0b5d59859d41f + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- !!merge <<: *ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q8" + variants: [] + description: | + Ornith-1.5-35B-A3B in the higher-quality Q8_0 GGUF format, with the shared + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q8_0.gguf + sha256: de46c4baf4b4dd85ea438bb0f757f21c38841a353506579979bba114311658c3 + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: From ccb9a0a088fd5630ffcd94d88c5ec7321728e7ed Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:55:43 +0200 Subject: [PATCH 28/75] chore: :arrow_up: Update 0xShug0/audio.cpp to `d25ffac094a9d5a240940b4955ea79ad9b7b4c78` (#11710) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index bee144fbc..2462a4f40 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03 +AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From fa9ffc181ce2b0e7682e9b600061cb603335b3d0 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:57:12 +0200 Subject: [PATCH 29/75] chore: :arrow_up: Update ggml-org/llama.cpp to `f280b26983ad0fdb705a0d9ebf0503e76f2899b0` (#11646) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): adapt to the common JSON API The llama.cpp bump replaces its nlohmann JSON alias with common_json. Update the gRPC adapter for the new exception, iterator, conversion, and container APIs. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(turboquant): adapt the JSON exception type The shared gRPC source now follows the upstream common_json API. The TurboQuant fork still exposes nlohmann JSON and cannot compile the new exception type. Translate that exception in the fork-specific source patch so both llama.cpp variants compile from the shared adapter. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(bonsai): adapt the JSON exception type The shared gRPC source uses upstream's common_json wrapper. The Bonsai fork still exposes nlohmann JSON and cannot compile that exception type.\n\nTranslate the exception in the fork-specific preparation step and verify that repeated preparation stays idempotent.\n\nAssisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(llama-cpp): let prepare register gRPC The score patch duplicated the gRPC CMake registration that prepare.sh already owns. Its stale context rejects the current upstream tools file on Darwin before compilation starts. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 + backend/cpp/bonsai/patch-grpc-server.sh | 24 ++++++++ backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 55 +++++++++---------- backend/cpp/llama-cpp/message_content.h | 7 ++- .../0001-add-server-task-type-score.patch | 12 +--- backend/cpp/turboquant/patch-grpc-server.sh | 14 +++++ scripts/build/bonsai-json-compat_test.sh | 25 +++++++++ 8 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 backend/cpp/bonsai/patch-grpc-server.sh create mode 100644 scripts/build/bonsai-json-compat_test.sh diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 96171e6a9..a3cdf980f 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -41,6 +41,7 @@ define bonsai-build # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:$(1)$(RESET)) @@ -79,6 +80,7 @@ bonsai-cpu-all: # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET)) diff --git a/backend/cpp/bonsai/patch-grpc-server.sh b/backend/cpp/bonsai/patch-grpc-server.sh new file mode 100644 index 000000000..aa9b23a50 --- /dev/null +++ b/backend/cpp/bonsai/patch-grpc-server.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +SRC=$1 +if [[ ! -f "$SRC" ]]; then + echo "grpc-server.cpp not found at $SRC" >&2 + exit 2 +fi + +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the Bonsai JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> Bonsai JSON exception patch OK" +else + echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping" +fi diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 41861f5f2..d9e248adf 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd +LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 171ae0483..7c1a3d320 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const } else { SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str()); } - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what()); SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str()); } @@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str()); } SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { // If parsing fails, treat as string data["tool_choice"] = predict->toolchoice(); SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str()); @@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const // Add to data - llama.cpp server expects it as an object (map) data["logit_bias"] = logit_bias_json; SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what()); } } @@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const }); } - data["stop"] = predict->stopprompts(); + data["stop"] = json::array(); + for (const auto & stop : predict->stopprompts()) { + data["stop"].push_back(stop); + } // data["n_probs"] = predict->nprobs(); //TODO: images, @@ -1795,7 +1798,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -1988,7 +1991,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2074,30 +2077,27 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - for (const auto &img : *images_data) + for (const auto &img : data.at("image_data")) { auto decoded_data = base64_decode(img["data"].get()); files.push_back(decoded_data); } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -2370,7 +2370,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -2563,7 +2563,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2649,11 +2649,10 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl; - for (const auto &img : *images_data) + std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl; + for (const auto &img : data.at("image_data")) { std::cout << "[PREDICT] Processing image" << std::endl; auto decoded_data = base64_decode(img["data"].get()); @@ -2661,20 +2660,18 @@ public: } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -3005,7 +3002,7 @@ public: } // Collect responses - json responses = json::array(); + std::vector responses; for (auto & res : all_results.results) { GGML_ASSERT(dynamic_cast(res.get()) != nullptr); responses.push_back(res->to_json()); @@ -3018,7 +3015,7 @@ public: // Crop results by request.top_n if specified int top_n = request->top_n(); if (top_n > 0 && top_n < static_cast(responses.size())) { - responses = json(responses.begin(), responses.begin() + top_n); + responses.resize(top_n); } // Set usage information backend::Usage* usage = rerankResult->mutable_usage(); diff --git a/backend/cpp/llama-cpp/message_content.h b/backend/cpp/llama-cpp/message_content.h index 4c7317ecd..0b70c8b56 100644 --- a/backend/cpp/llama-cpp/message_content.h +++ b/backend/cpp/llama-cpp/message_content.h @@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role, // (#7528). A multimodal user message legitimately carries a typed-part array // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the // streaming and non-streaming paths so this invariant cannot drift between them. -inline void normalize_template_message(nlohmann::ordered_json& msg) { +template +inline void normalize_template_message(Json& msg) { if (!msg.contains("content")) { msg["content"] = ""; // templates expect the field to exist return; } - nlohmann::ordered_json& content = msg["content"]; + auto& content = msg["content"]; const std::string role = (msg.contains("role") && msg["role"].is_string()) - ? msg["role"].get() + ? msg["role"].template get() : std::string(); if (content.is_null()) { content = ""; // #7324: null would crash content[:N] slicing diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index f056d47ce..253e8da5f 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch --- common/common.cpp | 6 +- common/common.h | 3 + - tools/CMakeLists.txt | 1 + tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++- tools/server/server-task.h | 47 +++++ - 5 files changed, 406 insertions(+), 9 deletions(-) + 4 files changed, 405 insertions(+), 9 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2e3f14c..0cec0dc 100644 @@ -42,15 +41,6 @@ index 878534d..4001df2 100644 int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t n_outputs_max_per_seq = 1; // max outputs per sequence -diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt -index 780df32..1d2fe8f 100644 ---- a/tools/CMakeLists.txt -+++ b/tools/CMakeLists.txt -@@ -41,3 +41,4 @@ else() - add_subdirectory(fit-params) - add_subdirectory(results) - endif() -+add_subdirectory(grpc-server) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3b5f6a1..d0e18e6 100644 --- a/tools/server/server-context.cpp diff --git a/backend/cpp/turboquant/patch-grpc-server.sh b/backend/cpp/turboquant/patch-grpc-server.sh index fa11897dd..8d1e35578 100755 --- a/backend/cpp/turboquant/patch-grpc-server.sh +++ b/backend/cpp/turboquant/patch-grpc-server.sh @@ -8,6 +8,8 @@ # so the grpc-server option parser skips the two references to # common_params::checkpoint_min_step (the default and the option handler). # That field does not exist in the fork yet; drop this once it does. +# 3. Use nlohmann's parse_error type in JSON catch clauses because the fork +# predates upstream's common_json_error wrapper. # # The fork used to lag upstream on the whole common_params_speculative refactor # (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and @@ -100,4 +102,16 @@ else echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK" fi +# 3. The shared source follows current upstream and catches common_json_error. +# TurboQuant still exposes nlohmann::json directly, so its equivalent parse +# failures use json::parse_error instead. +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the TurboQuant JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> TurboQuant JSON exception patch OK" +else + echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping" +fi + echo "==> all patches applied" diff --git a/scripts/build/bonsai-json-compat_test.sh b/scripts/build/bonsai-json-compat_test.sh new file mode 100644 index 000000000..c0cfc4f88 --- /dev/null +++ b/scripts/build/bonsai-json-compat_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +PATCHER="$ROOT/backend/cpp/bonsai/patch-grpc-server.sh" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/grpc-server.cpp" <<'EOF' +try { + json::parse("{"); +} catch (const common_json_error& e) { +} +EOF + +bash "$PATCHER" "$WORK/grpc-server.cpp" +grep -q 'catch (const json::parse_error& e)' "$WORK/grpc-server.cpp" +! grep -q 'common_json_error' "$WORK/grpc-server.cpp" + +# A repeated preparation pass must not change the generated source. +cp "$WORK/grpc-server.cpp" "$WORK/once.cpp" +bash "$PATCHER" "$WORK/grpc-server.cpp" +cmp "$WORK/once.cpp" "$WORK/grpc-server.cpp" + +echo "PASS: Bonsai uses its fork-compatible JSON exception" From edabdf950104bf5949825b98e8001b55983ac31e Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 25 Aug 2026 17:36:02 +0200 Subject: [PATCH 30/75] feat(gallery): add Ornith 1.5 397B variants (#11716) * feat(gallery): add Ornith 1.5 397B variants Add the official Q4_K_M and Q8_0 GGUF builds with their shared BF16 vision projector. Assisted-by: Codex:gpt-5 * feat(gallery): resolve Ornith variant ordering\n\nKeep the 35B entries from master next to the 397B variants.\n\nAssisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 35392e405..10690321f 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -694,6 +694,102 @@ - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- &ornith-1-5-397b + name: "ornith-1.5-397b-q4" + variants: + - model: ornith-1.5-397b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-397B + - https://huggingface.co/ornith-ai/Ornith-1.5-397B-GGUF + description: | + Ornith-1.5-397B is Ornith AI's MIT-licensed flagship mixture-of-experts + model for agentic coding, reasoning, repository-level tasks, and tool use. + It supports text and image input with a context window of 262K tokens. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. A + higher-quality Q8_0 model is available as a variant. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q4_K_M.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q4_K_M.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/Ornith-1.5-397B-Q4_K_M.gguf + sha256: c7775e6fae1a47619c199c81b865df9014e5d724509a098377ce1c84744b6552 + - filename: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/mmproj-Ornith-1.5-397B-BF16.gguf + sha256: 9da8c035659d9782b80c2ca6bebdb3befb98a2d7c5a3ac5c001d5a9f02f76fdc +- !!merge <<: *ornith-1-5-397b + name: "ornith-1.5-397b-q8" + variants: [] + description: | + Ornith-1.5-397B in the higher-quality Q8_0 GGUF format, with the shared + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q8_0.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-397b/Ornith-1.5-397B-Q8_0.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/Ornith-1.5-397B-Q8_0.gguf + sha256: 1e033a38f099a5c125e3cf762ef93b6f3db071e18afc55dea6c265c4d0768d8e + - filename: llama-cpp/mmproj/ornith-1.5-397b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-397B-GGUF/mmproj-Ornith-1.5-397B-BF16.gguf + sha256: 9da8c035659d9782b80c2ca6bebdb3befb98a2d7c5a3ac5c001d5a9f02f76fdc - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: From 33dafe37ab9aff35ed3f0876a4b70138c7840793 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:06 +0200 Subject: [PATCH 31/75] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `08b500b958a3f1102e6500e5c425e65517d6fb7e` (#11726) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index f3ff79a7f..e05a7a00f 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340 +IK_LLAMA_VERSION?=08b500b958a3f1102e6500e5c425e65517d6fb7e LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From f63f11eb869609a52fd5c58a2a80e66a8f0c6c17 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:19 +0200 Subject: [PATCH 32/75] chore: :arrow_up: Update 0xShug0/audio.cpp to `c79e58899bf13db4d78fd06372da23cc13f55b28` (#11722) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 2462a4f40..f2c742115 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78 +AUDIO_CPP_VERSION?=c79e58899bf13db4d78fd06372da23cc13f55b28 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 5755898e5732b227bc39239e870b4f856f3589fc Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:01:32 +0200 Subject: [PATCH 33/75] chore: :arrow_up: Update ggml-org/whisper.cpp to `978113305b2ead22249b881deafa131dc8884911` (#11711) :arrow_up: Update ggml-org/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/whisper/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 214fba777..8b0f02d64 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=233fe1fc9b48a09e361d3594520838ca266537fe +WHISPER_CPP_VERSION?=978113305b2ead22249b881deafa131dc8884911 SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From f28e8b24e65463178fb3da7430679454d03d180e Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:57:36 +0800 Subject: [PATCH 34/75] fix(ollama): accept :latest tag on model lookup (#11732) /api/tags appends :latest to untagged names, but chat and the other model endpoints looked the tagged name up as-is and 404'd. Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- core/http/endpoints/ollama/models_test.go | 14 ++++++++++++++ core/http/middleware/request.go | 6 ++++++ core/http/middleware/request_test.go | 14 ++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/core/http/endpoints/ollama/models_test.go b/core/http/endpoints/ollama/models_test.go index b13cf59a0..c4d0d6b5e 100644 --- a/core/http/endpoints/ollama/models_test.go +++ b/core/http/endpoints/ollama/models_test.go @@ -146,6 +146,20 @@ parameters: Expect(resp.Details.Format).To(Equal("gguf")) Expect(resp.Details.Families).ToNot(BeEmpty()) }) + + It("looks up the model when the Ollama :latest tag is included", func() { + writeConfig("chat", ` +name: chat +backend: llama-cpp +template: + chat: "{{ .Input }}" +parameters: + model: Llama-3-8B-Q4_K_M.gguf +`) + resp := callShow("chat:latest") + Expect(resp.Details.Format).To(Equal("gguf")) + Expect(resp.Capabilities).To(ContainElement("completion")) + }) }) Describe("ListModelsEndpoint", func() { diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 1599ef05c..080a0b73c 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -141,6 +141,12 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR } modelName := input.ModelName(nil) + // Ollama-compat /api/tags appends ":latest" to untagged names. + // Strip it for lookup so the listed name works on /api/chat, + // /v1/chat/completions, and the other model-bearing endpoints. + if strings.HasSuffix(modelName, ":latest") { + modelName = strings.TrimSuffix(modelName, ":latest") + } cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(modelName, re.applicationConfig) if err != nil { diff --git a/core/http/middleware/request_test.go b/core/http/middleware/request_test.go index 1b00c7f02..afaf8d9c8 100644 --- a/core/http/middleware/request_test.go +++ b/core/http/middleware/request_test.go @@ -82,6 +82,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(resp.Error.Message).To(ContainSubstring("not found")) Expect(resp.Error.Type).To(Equal("invalid_request_error")) }) + + It("still 404s when :latest is appended to an unknown model", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"nonexistent-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) }) Context("when the model exists as a config file", func() { @@ -97,6 +104,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(rec.Code).To(Equal(http.StatusOK)) }) + + It("accepts the Ollama :latest tag that /api/tags appends", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"test-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) }) Context("when the model exists as a pre-loaded config", func() { From aea477932d88e6ba6de24f30e01135d0f1764799 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:57:58 +0200 Subject: [PATCH 35/75] chore(model-gallery): :arrow_up: update checksum (#11730) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 10690321f..f3a069550 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -45,8 +45,8 @@ use_tokenizer_template: true files: - filename: llama-cpp/models/Qwen3.8-27B-DFlash2-Q4_K_M/Qwen3.8-27B-DFlash2-Q4_K_M.gguf - sha256: 18a380efc9b7ed8d88677fc895f5c11ae170653434ee378f7348f715c14d0594 uri: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q4_K_M.gguf + sha256: 1a25c56858e1ebe93f2718ac1d49d1151f9323325c1bbfd6209370f4db131ebd - name: "huihui-qwen3.8-27b-abliterated" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From e7b83ef7c0014036c14de82509d4b7e671e0b5db Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:58:17 +0200 Subject: [PATCH 36/75] Fix flaky "tests-apple" CI job in modeladmin test suite (#11717) * Initial plan * tests: raise default Eventually timeout for modeladmin suite to fix flaky macOS CI Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- core/services/modeladmin/modeladmin_suite_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/services/modeladmin/modeladmin_suite_test.go b/core/services/modeladmin/modeladmin_suite_test.go index a4332e395..b0a053167 100644 --- a/core/services/modeladmin/modeladmin_suite_test.go +++ b/core/services/modeladmin/modeladmin_suite_test.go @@ -2,6 +2,7 @@ package modeladmin import ( "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -9,5 +10,12 @@ import ( func TestModelAdmin(t *testing.T) { RegisterFailHandler(Fail) + // Several specs in this suite coordinate goroutines through + // Eventually/Consistently on unbuffered-ish channels (e.g. the + // blockingRevisionLifecycle helper). Gomega's 1s default timeout can be + // too tight on slower or loaded CI runners (notably macOS runners), + // causing spurious "Timed out after 1.005s" failures even though the + // goroutines eventually make progress. Give them more headroom. + SetDefaultEventuallyTimeout(5 * time.Second) RunSpecs(t, "modeladmin test suite") } From 5c08ec0382eb644076079affd4b46e87e6c01a7d Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:58:37 +0200 Subject: [PATCH 37/75] chore: :arrow_up: Update ggml-org/llama.cpp to `eab8ee41f889ef7823af517e8098fb8a9b3cf601` (#11724) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index d9e248adf..ae8cff292 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0 +LLAMA_VERSION?=eab8ee41f889ef7823af517e8098fb8a9b3cf601 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 15f12074ca662708b193084fd35753bc2d38c0d4 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:59:12 +0200 Subject: [PATCH 38/75] chore: :arrow_up: Update leejet/stable-diffusion.cpp to `50d640568388f876b0d63ee6ddb6bc86d997ec64` (#11725) :arrow_up: Update leejet/stable-diffusion.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/stablediffusion-ggml/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 1c1552074..4aa9b09d0 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=97d2990807fe6d558e395f8764198d7c7e7b411c +STABLEDIFFUSION_GGML_VERSION?=50d640568388f876b0d63ee6ddb6bc86d997ec64 CMAKE_ARGS+=-DGGML_MAX_NAME=128 From 5dab4fcde97773f99fdf6c7c3cf7d4f37c5e45f9 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 26 Aug 2026 09:01:55 +0200 Subject: [PATCH 39/75] feat(gallery): add Granite 4.2 variants (#11719) Add the official IBM Q4_K_M and Q8_0 GGUF builds for the 3B, 8B, and 30B Granite 4.2 models. Assisted-by: Codex:gpt-5.6-sol Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 211 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index f3a069550..b1b41fe93 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,215 @@ --- +- &granite-4-2-3b + name: "granite-4.2-3b-q4" + variants: + - model: granite-4.2-3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-3b + - https://huggingface.co/ibm-granite/granite-4.2-3b-GGUF + description: | + IBM Granite 4.2 3B is a compact multilingual reasoning model for chat, + coding, long-context tasks, and tool use. This entry uses the Q4_K_M + GGUF; a higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q4_K_M.gguf + sha256: 20e436143017578687f7f848225cc6c6038126c84149192229c7dff6e4e0f427 +- !!merge <<: *granite-4-2-3b + name: "granite-4.2-3b-q8" + variants: [] + description: | + IBM Granite 4.2 3B in the higher-fidelity Q8_0 GGUF format. It is a + compact multilingual reasoning model for chat, coding, and tool use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q8_0.gguf + sha256: 9e97320b131445ab8d9098cafb48001e9925d879e71486a8af4db4c803c55394 +- &granite-4-2-8b + name: "granite-4.2-8b-q4" + variants: + - model: granite-4.2-8b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-8b + - https://huggingface.co/ibm-granite/granite-4.2-8b-GGUF + description: | + IBM Granite 4.2 8B is a multilingual reasoning model for chat, coding, + long-context tasks, and tool use. This entry uses the Q4_K_M GGUF; a + higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-8b-GGUF/granite-4.2-8b-Q4_K_M.gguf + sha256: 16a9369d0805f80b7377d25d87f937a90c05dc04ad79173a52001e42c9aab311 +- !!merge <<: *granite-4-2-8b + name: "granite-4.2-8b-q8" + variants: [] + description: | + IBM Granite 4.2 8B in the higher-fidelity Q8_0 GGUF format. It is a + multilingual reasoning model for chat, coding, and tool use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-8b/granite-4.2-8b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-8b-GGUF/granite-4.2-8b-Q8_0.gguf + sha256: fb66ad5750680c77c76b9dc095961375f14fc88ba15fd5fc084ef6e6701bad77 +- &granite-4-2-30b + name: "granite-4.2-30b-q4" + variants: + - model: granite-4.2-30b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ibm-granite/granite-4.2-30b + - https://huggingface.co/ibm-granite/granite-4.2-30b-GGUF + description: | + IBM Granite 4.2 30B is the family's flagship multilingual reasoning model + for chat, coding, long-context tasks, and tool use. This entry uses the + Q4_K_M GGUF; a higher-fidelity Q8_0 build is available as a variant. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - granite + - multilingual + - reasoning + - thinking + - coding + - tools + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q4_K_M.gguf + uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q4_K_M.gguf + sha256: f299dace85d77ee0e24ca0b6720bd211b1f4cd1f65b9c0a2111e79692e29c9be +- !!merge <<: *granite-4-2-30b + name: "granite-4.2-30b-q8" + variants: [] + description: | + IBM Granite 4.2 30B in the higher-fidelity Q8_0 GGUF format. It is the + family's flagship multilingual reasoning model for chat, coding, and tool + use. + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf + uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q8_0.gguf + sha256: 005b0933353e9ba219b26e2667705bdb8dbc74eb50e4a4e6cb70fca108710f81 - name: "qwen3.8-27b-dflash2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From bbd3ab5a1458498895fa142b633579ad3436471b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 26 Aug 2026 09:02:31 +0200 Subject: [PATCH 40/75] feat(gallery): add Tiel-Coder 35B variants (#11723) Add Q4_K_XL, MTP Q4_K_XL, and Q8_K_XL builds with their BF16 vision projectors. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 134 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index b1b41fe93..4d8fe4d31 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -905,6 +905,140 @@ - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- &tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q4" + variants: + - model: tiel-coder-35b-a3b-q4-mtp + - model: tiel-coder-35b-a3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF + description: | + Tiel-Coder-35B-A3B is a 35B-parameter mixture-of-experts model for coding, + reasoning, tool use, and vision tasks. This default entry uses the + Q4_K_XL GGUF and BF16 vision projector. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/Tiel-Coder-35B-A3B-UD-Q4_K_XL.gguf + sha256: 9286a94c453c6a40ad51982c3dc88df4bba32fee9efad06e4588c83c059cf17c + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 +- !!merge <<: *tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q4-mtp" + variants: [] + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP + description: | + Tiel-Coder-35B-A3B in Q4_K_XL format with MTP speculative decoding and a + BF16 vision projector. + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - mtp + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b-mtp/mmproj-BF16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b-mtp/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b-mtp/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf + sha256: 10960d1d6477b08ed36a0e542e571b473022023c25b8315b0cf8c33c57e98ccd + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b-mtp/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 +- !!merge <<: *tiel-coder-35b-a3b + name: "tiel-coder-35b-a3b-q8" + variants: [] + description: | + Tiel-Coder-35B-A3B in the higher-quality Q8_K_XL GGUF format, with the + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + model: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/tiel-coder-35b-a3b/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/Tiel-Coder-35B-A3B-UD-Q8_K_XL.gguf + sha256: 883faacea54421f80f5d2713b344c69fbd2c76093a8cc4e2694dbb6958c4d699 + - filename: llama-cpp/mmproj/tiel-coder-35b-a3b/mmproj-BF16.gguf + uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF/mmproj-BF16.gguf + sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837 - &ornith-1-5-397b name: "ornith-1.5-397b-q4" variants: From fa19b08f354ac54538ede9d093be60b263e35a6f Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:16:49 +0200 Subject: [PATCH 41/75] chore: :arrow_up: Update mudler/vllm.cpp to `6738e0b4639199f3ff0998815e4d32bfa7fe5be2` (#11647) * :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(vllm-cpp): mirror ABI v23 The new engine pin reports ABI v23 and appends mmproj_path to vllm_model_params. LocalAI still declares v21, so the build-time ABI guard rejects every backend build. Grow the Go mirror by the appended pointer and update its offset checks. ABI v23 adds a video function but does not change the mirrored text structs. Assisted-by: Codex:gpt-5.6 [systematic-debugging] --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot --- backend/go/vllm-cpp/Makefile | 2 +- backend/go/vllm-cpp/govllmcpp.go | 5 +++-- backend/go/vllm-cpp/vllmcpp_test.go | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 754ce5560..d403319f5 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=438305e1577768ec0f75729456a4c8b9f425e2ee +VLLM_CPP_VERSION?=6738e0b4639199f3ff0998815e4d32bfa7fe5be2 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index b880105af..20e587a6b 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v21). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 21 +const abiVersion = 23 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -83,6 +83,7 @@ type cModelParams struct { LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19) _ [4]byte LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) + MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 681ca4d4e..55f6e211a 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v21) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(21)) + Expect(abiVersion).To(Equal(23)) }) It("cModelParams matches vllm_model_params", func() { @@ -51,7 +51,8 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128))) + Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { From 6f6ddba746a2d07172f21b44a9d646bef00edbad Mon Sep 17 00:00:00 2001 From: Szymon Podeszwa <2962046+sz-po@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:33:37 +0200 Subject: [PATCH 42/75] fix(deps): bump go-m1cpu to v0.2.2 to fix SIGSEGV on Apple M5 (#11736) go-m1cpu v0.1.6 runs its cgo initialiser from a package init(), where getFrequency() dereferences the CFTypeRef returned by IORegistryEntryCreateCFProperty without a NULL check. On Apple M5 the pmgr IORegistry node does not expose voltage-states5-sram / voltage-states1-sram in the shape v0.1.6 expects, so the call returns NULL and CFDataGetLength(NULL) faults before main() runs. Every command dies, including local-ai --version. The package is linked indirectly: cmd/local-ai reaches gopsutil/v3/{process,disk}, which pull in gopsutil/v3/cpu on darwin, which calls m1cpu.IsAppleSilicon() and m1cpu.PCoreHz(). v0.2.2 adds the missing NULL guard and moves the IORegistry probe out of init() behind a lazy sync.Once. The exported Go API is unchanged and the non-darwin stub is byte-identical, so gopsutil/v3 compiles against it untouched and no other platform is affected. Bumping gopsutil/v3 is not an alternative: v3.24.5 is the final v3 release, so the v3 line will never carry this fix. Fixes #11735 Assisted-by: Claude:claude-opus-5 Signed-off-by: Szymon Podeszwa <2962046+sz-po@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5418b351d..5ed7e0b51 100644 --- a/go.mod +++ b/go.mod @@ -498,7 +498,7 @@ require ( github.com/quic-go/quic-go v0.59.0 // indirect github.com/quic-go/webtransport-go v0.10.0 // indirect github.com/rivo/uniseg v0.4.7 - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shoenig/go-m1cpu v0.2.2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d // indirect diff --git a/go.sum b/go.sum index 6169840bd..6a0be2c19 100644 --- a/go.sum +++ b/go.sum @@ -1263,8 +1263,11 @@ github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfx github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/go-m1cpu v0.2.2 h1:4nc55oVv7nygGnfI9bhLCLzUEs4794y0Bkqx4q2zy7Y= +github.com/shoenig/go-m1cpu v0.2.2/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= From 0a89fdb1d00a1e1e0cbbde9bd60807fa93b6ce51 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:18:18 +0200 Subject: [PATCH 43/75] chore(model-gallery): :arrow_up: update checksum (#11742) :arrow_up: Checksum updates in gallery/index.yaml Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/index.yaml b/gallery/index.yaml index 4d8fe4d31..b44c4b427 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1860,7 +1860,7 @@ files: - filename: llama-cpp/models/nemotron-3.5-lightning-30b-a3b/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf uri: huggingface://ggml-org/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q8_0.gguf - sha256: b0e25ce2d301930e706d549a59d18c5969dcec664fdf7466435287b03fefda36 + sha256: d4cc4c9fffaa356db8b49cfaba9233609cfb29b4dd89d827665210dc1cc8dbbb - &muse-glimmer-30b name: "muse-glimmer-30b" variants: From 74b885c31a3d62c5d3149ecd689dcbfc4130d515 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:12 +0200 Subject: [PATCH 44/75] chore: :arrow_up: Update PrismML-Eng/llama.cpp to `312bb2a93ea2bf798333fa859614fbf913ecb9e2` (#11740) :arrow_up: Update PrismML-Eng/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index a3cdf980f..522791d42 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=9ca265a57f85f2117942490f421f64a226dd9847 +BONSAI_VERSION?=312bb2a93ea2bf798333fa859614fbf913ecb9e2 LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= From 8712d37e2efd698af63d25c0e7554303a10c73c4 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:25 +0200 Subject: [PATCH 45/75] chore: :arrow_up: Update vllm-project/vllm cu130 wheel to `0.28.0` (#11741) :arrow_up: Update vllm-project/vllm cu130 wheel Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/python/vllm/requirements-cublas13-after.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/python/vllm/requirements-cublas13-after.txt b/backend/python/vllm/requirements-cublas13-after.txt index 34f4f50ff..b519b73fe 100644 --- a/backend/python/vllm/requirements-cublas13-after.txt +++ b/backend/python/vllm/requirements-cublas13-after.txt @@ -3,8 +3,8 @@ # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match # so uv consults this index alongside PyPI. ---extra-index-url https://wheels.vllm.ai/0.27.1/cu130 +--extra-index-url https://wheels.vllm.ai/0.28.0/cu130 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh), # which pins this exact vLLM version. Bumping vllm here means coordinating with a # vllm-metal release that supports the new version, or macOS/Metal builds break. -vllm==0.27.1 +vllm==0.28.0 From 460c22bff65b23f1765656c21d6aa79c6a0e4194 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:36 +0200 Subject: [PATCH 46/75] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `ef40550042973817ac391ca95a2ff041f512257b` (#11743) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index e05a7a00f..34fab9e6b 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=08b500b958a3f1102e6500e5c425e65517d6fb7e +IK_LLAMA_VERSION?=ef40550042973817ac391ca95a2ff041f512257b LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From 1b4c4853fb86a622fd657b1713efc7d70cf30bd2 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:53:49 +0200 Subject: [PATCH 47/75] chore: :arrow_up: Update ggml-org/llama.cpp to `925e1179947ea0c0ebfb0032df18af3a729822be` (#11744) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index ae8cff292..bf3e20aab 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=eab8ee41f889ef7823af517e8098fb8a9b3cf601 +LLAMA_VERSION?=925e1179947ea0c0ebfb0032df18af3a729822be LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From 1070cb12450328fa684de11717985fdec78e66aa Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:54:02 +0200 Subject: [PATCH 48/75] chore: :arrow_up: Update 0xShug0/audio.cpp to `db21cbdd60f3d2ff62114bc863781ff8073ac39b` (#11746) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index f2c742115..4e5391301 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=c79e58899bf13db4d78fd06372da23cc13f55b28 +AUDIO_CPP_VERSION?=db21cbdd60f3d2ff62114bc863781ff8073ac39b AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From e58dabf75f03a0b65f95d91ee06f9c8b1e16f226 Mon Sep 17 00:00:00 2001 From: "Plamen K. Kosseff" <333840+blackd@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:32:32 +0300 Subject: [PATCH 49/75] feat(ui): add 'Focus mode' option in chat settings to persistently toggle the sidebar auto-collapse (#11750) Assisted-by: Claude:claude-fable-5 Signed-off-by: Plamen K. Kosseff --- .../http/react-ui/public/locales/en/chat.json | 2 ++ core/http/react-ui/src/pages/Chat.jsx | 28 +++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/core/http/react-ui/public/locales/en/chat.json b/core/http/react-ui/public/locales/en/chat.json index f004cee56..f1c058790 100644 --- a/core/http/react-ui/public/locales/en/chat.json +++ b/core/http/react-ui/public/locales/en/chat.json @@ -33,6 +33,8 @@ "title": "Chat Settings", "manageMode": "Manage mode", "manageModeDesc": "Let this chat install models, switch backends, and edit configs by talking to LocalAI.", + "focusMode": "Focus mode", + "focusModeDesc": "Collapse the sidebar and slim the header while a conversation is active. Esc restores them temporarily; turn this off to keep the full layout.", "systemPrompt": "System Prompt", "systemPromptPlaceholder": "You are a helpful assistant...", "temperature": "Temperature", diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index 5866a2514..d799efba0 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -21,6 +21,8 @@ import { useOperations } from '../hooks/useOperations' import { relativeTime } from '../utils/format' import { copyToClipboard } from '../utils/clipboard' +const FOCUS_MODE_KEY = 'localai_chat_focus_mode' + function getLastMessagePreview(chat) { if (!chat.history || chat.history.length === 0) return '' for (let i = chat.history.length - 1; i >= 0; i--) { @@ -405,12 +407,20 @@ export default function Chat() { // Focus mode: once a conversation has at least one message we slim the // surrounding chrome (collapse the global app rail, fade non-essential // header items). Esc gives the user back the full chrome for the rest of - // this session. + // this session. The settings drawer offers a persistent opt-out. const isInConversation = (activeChat?.history?.length || 0) > 0 const [focusOverride, setFocusOverride] = useState(false) - const focusActive = isInConversation && !focusOverride + const [focusModeEnabled, setFocusModeEnabled] = useState(() => { + try { return localStorage.getItem(FOCUS_MODE_KEY) !== 'false' } catch (_) { return true } + }) + const focusActive = focusModeEnabled && isInConversation && !focusOverride const prevAppCollapseRef = useRef(null) + const toggleFocusMode = (next) => { + setFocusModeEnabled(next) + try { localStorage.setItem(FOCUS_MODE_KEY, String(next)) } catch (_) {} + } + const artifacts = useMemo( () => canvasMode ? extractCodeArtifacts(activeChat?.history, 'role', 'assistant') : [], [activeChat?.history, canvasMode] @@ -1110,6 +1120,20 @@ export default function Chat() { /> )} +
+
+ + {t('settings.focusMode')} + + + {t('settings.focusModeDesc')} + +
+ +