|
diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js
index 5fd5fdc5b..097bf6087 100644
--- a/core/http/react-ui/src/utils/api.js
+++ b/core/http/react-ui/src/utils/api.js
@@ -457,7 +457,10 @@ export const agentCollectionsApi = {
reset: (name, userId) => postJSON(`/api/agents/collections/${enc(name)}/reset${userQ(userId)}`),
deleteEntry: (name, entry, userId) => fetchJSON(`/api/agents/collections/${enc(name)}/entry/delete${userQ(userId)}`, { method: 'DELETE', body: JSON.stringify({ entry }), headers: { 'Content-Type': 'application/json' } }),
sources: (name, userId) => fetchJSON(`/api/agents/collections/${enc(name)}/sources${userQ(userId)}`),
- addSource: (name, url, interval, userId) => postJSON(`/api/agents/collections/${enc(name)}/sources${userQ(userId)}`, { url, update_interval: interval }),
+ addSource: (name, url, interval, userId) => postJSON(`/api/agents/collections/${enc(name)}/sources${userQ(userId)}`, {
+ url,
+ update_interval: interval === undefined ? undefined : Number(interval),
+ }),
removeSource: (name, url, userId) => fetchJSON(`/api/agents/collections/${enc(name)}/sources${userQ(userId)}`, { method: 'DELETE', body: JSON.stringify({ url }), headers: { 'Content-Type': 'application/json' } }),
}
diff --git a/core/schema/localai.go b/core/schema/localai.go
index e160badad..a40f90c8d 100644
--- a/core/schema/localai.go
+++ b/core/schema/localai.go
@@ -353,10 +353,12 @@ type FaceEmbedResponse struct {
// FaceRegisterRequest enrolls a face into the 1:N recognition store.
type FaceRegisterRequest struct {
BasicModelRequest
- Img string `json:"img"`
- Name string `json:"name"`
- Labels map[string]string `json:"labels,omitempty"`
- Store string `json:"store,omitempty"` // vector store model; empty = local-store default
+ RegisteredAt time.Time `json:"registered_at,omitempty"` // original enrollment time when replaying a saved embedding
+ Embedding []float32 `json:"embedding,omitempty"`
+ Img string `json:"img"`
+ Name string `json:"name"`
+ Labels map[string]string `json:"labels,omitempty"`
+ Store string `json:"store,omitempty"` // vector store model; empty = local-store default
}
type FaceRegisterResponse struct {
diff --git a/core/services/facerecognition/registry.go b/core/services/facerecognition/registry.go
index adc9d9200..ae781dbb2 100644
--- a/core/services/facerecognition/registry.go
+++ b/core/services/facerecognition/registry.go
@@ -56,5 +56,6 @@ type Match struct {
var (
ErrNotFound = errors.New("facerecognition: id not found")
ErrEmptyEmbedding = errors.New("facerecognition: embedding is empty")
+ ErrInvalidEmbedding = errors.New("facerecognition: embedding must be finite and nonzero")
ErrDimensionMismatch = errors.New("facerecognition: embedding dimension mismatch")
)
diff --git a/core/services/facerecognition/replay_test.go b/core/services/facerecognition/replay_test.go
new file mode 100644
index 000000000..21c965905
--- /dev/null
+++ b/core/services/facerecognition/replay_test.go
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: MIT
+
+package facerecognition
+
+import (
+ "context"
+ "encoding/json"
+ "math"
+ "sync"
+ "testing"
+ "time"
+
+ grpc "github.com/mudler/LocalAI/pkg/grpc"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ ggrpc "google.golang.org/grpc"
+)
+
+func TestEnrollmentReplay(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Enrollment replay") }
+
+type replayStore struct {
+ grpc.Backend
+ mu sync.Mutex
+ entries map[string][]byte
+}
+
+func (s *replayStore) StoresSet(_ context.Context, in *pb.StoresSetOptions, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i, k := range in.Keys {
+ b, _ := json.Marshal(k.Floats)
+ s.entries[string(b)] = append([]byte(nil), in.Values[i].Bytes...)
+ }
+ return &pb.Result{Success: true}, nil
+}
+
+var _ = Describe("Enrollment replay", func() {
+ It("keeps the identity across registry instances and a cleared store", func(ctx SpecContext) {
+ storage := &replayStore{entries: map[string][]byte{}}
+ newRegistry := func() Registry {
+ return NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0)
+ }
+ vector := []float32{1, 0, 0, 0}
+ meta := Metadata{Name: "Alice", RegisteredAt: time.Now().UTC()}
+ first, err := newRegistry().Register(ctx, vector, meta)
+ Expect(err).NotTo(HaveOccurred())
+ again, err := newRegistry().Register(ctx, vector, meta)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(again).To(Equal(first))
+ Expect(storage.entries).To(HaveLen(1))
+ storage.entries = map[string][]byte{}
+ restored, err := newRegistry().Register(ctx, vector, meta)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(restored).To(Equal(first))
+ Expect(storage.entries).To(HaveLen(1))
+ })
+ It("rejects zero and non-finite embeddings before writing", func(ctx SpecContext) {
+ for _, v := range [][]float32{{0, 0}, {float32(math.NaN()), 1}, {float32(math.Inf(1)), 1}} {
+ storage := &replayStore{entries: map[string][]byte{}}
+ reg := NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0)
+ _, err := reg.Register(ctx, v, Metadata{Name: "Alice"})
+ Expect(err).To(HaveOccurred())
+ Expect(storage.entries).To(BeEmpty())
+ }
+ })
+})
diff --git a/core/services/facerecognition/store_registry.go b/core/services/facerecognition/store_registry.go
index d4fd0d971..abf7ee1ac 100644
--- a/core/services/facerecognition/store_registry.go
+++ b/core/services/facerecognition/store_registry.go
@@ -2,8 +2,10 @@ package facerecognition
import (
"context"
+ "encoding/binary"
"encoding/json"
"fmt"
+ "math"
"sort"
"sync"
"time"
@@ -57,13 +59,32 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta
if r.dim != 0 && len(embedding) != r.dim {
return Metadata{}, fmt.Errorf("%w: expected %d, got %d", ErrDimensionMismatch, r.dim, len(embedding))
}
+ var norm float64
+ key := make([]byte, 4*len(embedding))
+ for i, value := range embedding {
+ if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) {
+ return Metadata{}, ErrInvalidEmbedding
+ }
+ norm += float64(value) * float64(value)
+ // The store treats negative and positive zero as the same key.
+ if value == 0 {
+ value = 0
+ }
+ binary.LittleEndian.PutUint32(key[i*4:], math.Float32bits(value))
+ }
+ if norm == 0 {
+ return Metadata{}, ErrInvalidEmbedding
+ }
backend, err := r.resolve(ctx, r.storeName)
if err != nil {
return Metadata{}, fmt.Errorf("facerecognition: resolve store: %w", err)
}
- meta.ID = uuid.NewString()
+ // The vector store upserts by the exact embedding. Derive the ID from the
+ // same key so replaying a saved vector preserves identity across replicas
+ // and after the in-memory store restarts.
+ meta.ID = uuid.NewSHA1(uuid.NewSHA1(uuid.NameSpaceOID, []byte(r.storeName)), key).String()
if meta.RegisteredAt.IsZero() {
meta.RegisteredAt = time.Now().UTC()
}
diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go
index 5bdabdfb5..8ad530f1b 100644
--- a/core/services/messaging/subjects.go
+++ b/core/services/messaging/subjects.go
@@ -467,6 +467,12 @@ func SubjectNodeFilesStage(nodeID string) string {
return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.stage"
}
+// SubjectNodeFilesRelease tells a serve-backend node to evict one request's ephemeral cache keys.
+// Reply: {error}
+func SubjectNodeFilesRelease(nodeID string) string {
+ return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.release"
+}
+
// SubjectNodeFilesTemp tells a serve-backend node to allocate a temp file.
// Reply: {local_path, error}
func SubjectNodeFilesTemp(nodeID string) string {
diff --git a/core/services/nodes/file_stager.go b/core/services/nodes/file_stager.go
index c5ee38556..144c93165 100644
--- a/core/services/nodes/file_stager.go
+++ b/core/services/nodes/file_stager.go
@@ -1,6 +1,11 @@
package nodes
-import "context"
+import (
+ "context"
+ "fmt"
+ "path"
+ "strings"
+)
// FileStager abstracts file transfer between frontend and backend nodes
// in distributed mode. Two implementations exist:
@@ -29,7 +34,59 @@ type FileStager interface {
// StageRemoteToStore uploads a remote file to shared storage.
StageRemoteToStore(ctx context.Context, nodeID, remotePath, key string) error
+ // ReleaseRemote removes one ephemeral key from the remote node.
+ ReleaseRemote(ctx context.Context, nodeID, key string) error
+
// ListRemoteDir returns relative file paths within a directory on the remote node.
// keyPrefix is a storage-style key prefix (e.g. "models/mymodel").
ListRemoteDir(ctx context.Context, nodeID, keyPrefix string) ([]string, error)
}
+
+// RequestFileReleaser removes all ephemeral keys staged for one inference in
+// one transport operation. FileStagingClient falls back to ReleaseRemote for
+// stagers that do not implement this optional rolling-upgrade extension.
+type RequestFileReleaser interface {
+ ReleaseRemoteRequest(ctx context.Context, nodeID, requestID string, keys []string) error
+}
+
+func validateEphemeralRequestRelease(requestID string, keys []string) error {
+ if err := validateEphemeralRequestID(requestID); err != nil {
+ return err
+ }
+ if len(keys) == 0 {
+ return fmt.Errorf("release batch must contain at least one key")
+ }
+ for _, key := range keys {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ return err
+ }
+ parts := strings.Split(key, "/")
+ if parts[2] != requestID {
+ return fmt.Errorf("release batch mixes request IDs %q and %q", requestID, parts[2])
+ }
+ }
+ return nil
+}
+
+func validateEphemeralRequestID(requestID string) error {
+ if requestID == "" || strings.ContainsAny(requestID, "/\\") || path.Clean(requestID) != requestID || requestID == "." || requestID == ".." {
+ return fmt.Errorf("invalid ephemeral request ID %q", requestID)
+ }
+ return nil
+}
+
+func validateEphemeralReleaseKey(key string) error {
+ if strings.Contains(key, "\\") || path.Clean(key) != key {
+ return fmt.Errorf("invalid ephemeral key %q", key)
+ }
+ parts := strings.Split(key, "/")
+ if len(parts) != 4 || parts[0] != "ephemeral" {
+ return fmt.Errorf("release key %q must identify one file below ephemeral/", key)
+ }
+ for _, part := range parts[1:] {
+ if part == "" || part == "." || part == ".." {
+ return fmt.Errorf("invalid ephemeral key %q", key)
+ }
+ }
+ return nil
+}
diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go
index 79047aad6..0a64db5ab 100644
--- a/core/services/nodes/file_stager_http.go
+++ b/core/services/nodes/file_stager_http.go
@@ -1,6 +1,7 @@
package nodes
import (
+ "bytes"
"context"
"crypto/sha256"
"encoding/hex"
@@ -10,6 +11,7 @@ import (
"io"
"net"
"net/http"
+ "net/url"
"os"
"path/filepath"
"strconv"
@@ -81,6 +83,86 @@ func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token st
}
}
+// ReleaseRemote removes one exact ephemeral key from a backend node.
+func (h *HTTPFileStager) ReleaseRemote(ctx context.Context, nodeID, key string) error {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ return err
+ }
+ addr, err := h.httpAddrFor(nodeID)
+ if err != nil {
+ return fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err)
+ }
+ releaseURL := (&url.URL{Scheme: "http", Host: addr, Path: "/v1/files/" + key}).String()
+ req, err := http.NewRequestWithContext(ctx, http.MethodDelete, releaseURL, nil)
+ if err != nil {
+ return fmt.Errorf("creating release request for %q: %w", key, err)
+ }
+ if h.token != "" {
+ req.Header.Set("Authorization", "Bearer "+h.token)
+ }
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return fmt.Errorf("releasing %q from node %s: %w", key, nodeID, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ return fmt.Errorf("releasing %q from node %s: status %d: %s", key, nodeID, resp.StatusCode, strings.TrimSpace(string(body)))
+ }
+ return nil
+}
+
+// ReleaseRemoteRequest removes one inference's staged inputs with one HTTP
+// request. Older workers return 404 for the batch endpoint, so the client
+// retries through the exact-key API during rolling upgrades.
+func (h *HTTPFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, requestID string, keys []string) error {
+ if err := validateEphemeralRequestRelease(requestID, keys); err != nil {
+ return err
+ }
+ addr, err := h.httpAddrFor(nodeID)
+ if err != nil {
+ return fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err)
+ }
+ payload, err := json.Marshal(struct {
+ RequestID string `json:"request_id"`
+ }{RequestID: requestID})
+ if err != nil {
+ return fmt.Errorf("encoding request release: %w", err)
+ }
+ releaseURL := (&url.URL{Scheme: "http", Host: addr, Path: "/v1/files-release"}).String()
+ req, err := http.NewRequestWithContext(ctx, http.MethodDelete, releaseURL, bytes.NewReader(payload))
+ if err != nil {
+ return fmt.Errorf("creating request release: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ if h.token != "" {
+ req.Header.Set("Authorization", "Bearer "+h.token)
+ }
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return fmt.Errorf("releasing request inputs from node %s: %w", nodeID, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
+ return h.releaseRemoteKeys(ctx, nodeID, keys)
+ }
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ return fmt.Errorf("releasing request inputs from node %s: status %d: %s", nodeID, resp.StatusCode, strings.TrimSpace(string(body)))
+ }
+ return nil
+}
+
+func (h *HTTPFileStager) releaseRemoteKeys(ctx context.Context, nodeID string, keys []string) error {
+ var releaseErrors []error
+ for _, key := range keys {
+ if err := h.ReleaseRemote(ctx, nodeID, key); err != nil {
+ releaseErrors = append(releaseErrors, err)
+ }
+ }
+ return errors.Join(releaseErrors...)
+}
+
func (h *HTTPFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, key string) (string, error) {
xlog.Debug("Staging file to remote node via HTTP", "node", nodeID, "localPath", localPath, "key", key)
@@ -90,7 +172,9 @@ func (h *HTTPFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, ke
}
// Probe: check if the remote already has the file with matching content hash.
- if remotePath, ok := h.probeExisting(ctx, addr, localPath, key); ok {
+ if remotePath, ok, probeErr := h.probeExisting(ctx, addr, localPath, key); probeErr != nil {
+ return "", fmt.Errorf("claiming existing file on node %s: %w", nodeID, probeErr)
+ } else if ok {
xlog.Info("Upload skipped (file already exists with matching hash)", "node", nodeID, "key", key, "remotePath", remotePath)
return remotePath, nil
}
@@ -439,14 +523,15 @@ func isTransientError(err error) bool {
// probeExisting sends a HEAD request to check if the remote already has the
// file with a matching SHA-256 hash. Returns the remote path and true if the
-// upload can be skipped. Any errors (including 405 from older servers) silently
-// fall through so the caller proceeds with a normal PUT.
-func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key string) (string, bool) {
+// upload can be skipped. HEAD and hash errors fall through to a normal PUT.
+// Matching ephemeral files are claimed first; a 404 or 405 claim response
+// identifies an older worker and also falls through to PUT.
+func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key string) (string, bool, error) {
url := fmt.Sprintf("http://%s/v1/files/%s", addr, key)
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
- return "", false
+ return "", false, nil
}
if h.token != "" {
req.Header.Set("Authorization", "Bearer "+h.token)
@@ -454,18 +539,18 @@ func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key
resp, err := h.client.Do(req)
if err != nil {
- return "", false
+ return "", false, nil
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- return "", false
+ return "", false, nil
}
remotePath := resp.Header.Get(HeaderLocalPath)
remoteHash := resp.Header.Get(HeaderContentSHA256)
if remotePath == "" || remoteHash == "" {
- return "", false
+ return "", false, nil
}
// A 200 with a content hash is proof the worker is alive and serving right
@@ -475,14 +560,53 @@ func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key
localHash, err := hashLocalCached(ctx, localPath)
if err != nil {
- return "", false
+ return "", false, nil
}
if localHash != remoteHash {
- return "", false
+ return "", false, nil
}
- return remotePath, true
+ if strings.HasPrefix(key, "ephemeral/") {
+ claimed, err := h.claimExisting(ctx, addr, key)
+ if err != nil {
+ return "", false, err
+ }
+ if !claimed {
+ return "", false, nil
+ }
+ }
+
+ return remotePath, true, nil
+}
+
+func (h *HTTPFileStager) claimExisting(ctx context.Context, addr, key string) (bool, error) {
+ claimURL := (&url.URL{
+ Scheme: "http",
+ Host: addr,
+ Path: "/v1/files/" + key,
+ RawQuery: "claim=1",
+ }).String()
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, claimURL, nil)
+ if err != nil {
+ return false, fmt.Errorf("creating claim request for %q: %w", key, err)
+ }
+ if h.token != "" {
+ req.Header.Set("Authorization", "Bearer "+h.token)
+ }
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return false, fmt.Errorf("claiming %q: %w", key, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
+ return false, nil
+ }
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ return false, fmt.Errorf("claiming %q: status %d: %s", key, resp.StatusCode, strings.TrimSpace(string(body)))
+ }
+ return true, nil
}
// hashChunkSize is how much of a file is hashed between activity ticks and
diff --git a/core/services/nodes/file_stager_release_test.go b/core/services/nodes/file_stager_release_test.go
new file mode 100644
index 000000000..519ef6287
--- /dev/null
+++ b/core/services/nodes/file_stager_release_test.go
@@ -0,0 +1,374 @@
+package nodes
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/mudler/LocalAI/core/services/messaging"
+ "github.com/mudler/LocalAI/core/services/storage"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type releaseTestSubscription struct{}
+
+func (releaseTestSubscription) Unsubscribe() error { return nil }
+
+type releaseTestMessaging struct {
+ subject string
+ payload []byte
+ onRequest func()
+ requestCalled bool
+ requestCount int
+ timeout time.Duration
+ replies [][]byte
+}
+
+func (m *releaseTestMessaging) Publish(string, any) error { return nil }
+func (m *releaseTestMessaging) Subscribe(string, func([]byte)) (messaging.Subscription, error) {
+ return releaseTestSubscription{}, nil
+}
+func (m *releaseTestMessaging) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) {
+ return releaseTestSubscription{}, nil
+}
+func (m *releaseTestMessaging) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) {
+ return releaseTestSubscription{}, nil
+}
+func (m *releaseTestMessaging) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) {
+ return releaseTestSubscription{}, nil
+}
+func (m *releaseTestMessaging) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) {
+ m.subject = subject
+ m.payload = append([]byte(nil), data...)
+ m.requestCalled = true
+ m.requestCount++
+ m.timeout = timeout
+ if m.onRequest != nil {
+ m.onRequest()
+ }
+ if m.requestCount <= len(m.replies) {
+ return append([]byte(nil), m.replies[m.requestCount-1]...), nil
+ }
+ return []byte(`{}`), nil
+}
+func (m *releaseTestMessaging) IsConnected() bool { return true }
+func (m *releaseTestMessaging) Close() {}
+
+var _ = Describe("File stager exact-key release", func() {
+ startReleaseServer := func(stagingDir, token string) (*HTTPFileStager, func()) {
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ Expect(err).NotTo(HaveOccurred())
+ server, err := StartFileTransferServerWithListener(
+ listener,
+ stagingDir,
+ GinkgoT().TempDir(),
+ GinkgoT().TempDir(),
+ token,
+ 0,
+ )
+ Expect(err).NotTo(HaveOccurred())
+ return NewHTTPFileStager(func(string) (string, error) {
+ return listener.Addr().String(), nil
+ }, token), func() {
+ Expect(server.Shutdown(context.Background())).To(Succeed())
+ }
+ }
+
+ It("transmits URL metacharacters as the exact key", func() {
+ stagingDir := GinkgoT().TempDir()
+ categoryDir := filepath.Join(stagingDir, "ephemeral", "request-id", "audio")
+ Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
+
+ key := "ephemeral/request-id/audio/name ?#%2F.wav"
+ exactPath := filepath.Join(categoryDir, "name ?#%2F.wav")
+ wrongPath := filepath.Join(categoryDir, "name ")
+ for _, path := range []string{exactPath, exactPath + hashSidecarSuffix, exactPath + targetSidecarSuffix, wrongPath} {
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+ }
+
+ stager, stop := startReleaseServer(stagingDir, "release-token")
+ DeferCleanup(stop)
+ Expect(stager.ReleaseRemote(context.Background(), "node-1", key)).To(Succeed())
+
+ Expect(exactPath).NotTo(BeAnExistingFile())
+ Expect(exactPath + hashSidecarSuffix).NotTo(BeAnExistingFile())
+ Expect(exactPath + targetSidecarSuffix).NotTo(BeAnExistingFile())
+ Expect(wrongPath).To(BeAnExistingFile())
+ })
+
+ It("is idempotent and prunes empty category and request directories", func() {
+ stagingDir := GinkgoT().TempDir()
+ path := filepath.Join(stagingDir, "ephemeral", "request-id", "audio", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+
+ stager, stop := startReleaseServer(stagingDir, "release-token")
+ DeferCleanup(stop)
+ for range 2 {
+ Expect(stager.ReleaseRemote(context.Background(), "node-1", "ephemeral/request-id/audio/input.wav")).To(Succeed())
+ }
+
+ Expect(filepath.Join(stagingDir, "ephemeral", "request-id", "audio")).NotTo(BeADirectory())
+ Expect(filepath.Join(stagingDir, "ephemeral", "request-id")).NotTo(BeADirectory())
+ Expect(filepath.Join(stagingDir, "ephemeral")).To(BeADirectory())
+ })
+
+ It("releases one request's HTTP inputs in one batch", func() {
+ stagingDir := GinkgoT().TempDir()
+ keys := []string{
+ "ephemeral/audio/request-id/input.wav",
+ "ephemeral/images/request-id/frame.jpg",
+ }
+ for _, key := range keys {
+ path := filepath.Join(stagingDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+ }
+
+ stager, stop := startReleaseServer(stagingDir, "release-token")
+ DeferCleanup(stop)
+ Expect(stager.ReleaseRemoteRequest(context.Background(), "node-1", "request-id", keys)).To(Succeed())
+
+ for _, key := range keys {
+ Expect(filepath.Join(stagingDir, filepath.FromSlash(key))).NotTo(BeAnExistingFile())
+ }
+ })
+
+ It("falls back to exact HTTP releases for an older worker", func() {
+ batchCalls := 0
+ exactCalls := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/v1/files-release" {
+ batchCalls++
+ http.NotFound(w, r)
+ return
+ }
+ if strings.HasPrefix(r.URL.Path, "/v1/files/") && r.Method == http.MethodDelete {
+ exactCalls++
+ w.WriteHeader(http.StatusNoContent)
+ return
+ }
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }))
+ DeferCleanup(server.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(server.URL, "http://"), nil
+ }, "")
+ keys := []string{
+ "ephemeral/audio/request-id/input.wav",
+ "ephemeral/images/request-id/frame.jpg",
+ }
+
+ Expect(stager.ReleaseRemoteRequest(context.Background(), "node-1", "request-id", keys)).To(Succeed())
+
+ Expect(batchCalls).To(Equal(1))
+ Expect(exactCalls).To(Equal(2))
+ })
+
+ It("rejects non-ephemeral and traversing keys before making a request", func() {
+ resolved := false
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ resolved = true
+ return "127.0.0.1:1", nil
+ }, "token")
+
+ for _, key := range []string{
+ "models/model.gguf",
+ "ephemeral/../models/model.gguf",
+ "ephemeral/request-id/../../model.gguf",
+ "/ephemeral/request-id/audio/input.wav",
+ } {
+ Expect(stager.ReleaseRemote(context.Background(), "node-1", key)).NotTo(Succeed(), key)
+ }
+ Expect(resolved).To(BeFalse())
+ })
+
+ It("rejects symlink escapes", func() {
+ stagingDir := GinkgoT().TempDir()
+ outsideDir := GinkgoT().TempDir()
+ outsidePath := filepath.Join(outsideDir, "input.wav")
+ Expect(os.WriteFile(outsidePath, []byte("keep"), 0640)).To(Succeed())
+ requestDir := filepath.Join(stagingDir, "ephemeral", "request-id")
+ Expect(os.MkdirAll(requestDir, 0750)).To(Succeed())
+ Expect(os.Symlink(outsideDir, filepath.Join(requestDir, "audio"))).To(Succeed())
+
+ stager, stop := startReleaseServer(stagingDir, "release-token")
+ DeferCleanup(stop)
+ Expect(stager.ReleaseRemote(context.Background(), "node-1", "ephemeral/request-id/audio/input.wav")).NotTo(Succeed())
+ Expect(outsidePath).To(BeAnExistingFile())
+ })
+
+ It("rejects symlinked files and sidecars without deleting their targets", func() {
+ for _, linkedName := range []string{"input.wav", "input.wav" + hashSidecarSuffix, "input.wav" + targetSidecarSuffix} {
+ stagingDir := GinkgoT().TempDir()
+ categoryDir := filepath.Join(stagingDir, "ephemeral", "request-id", "audio")
+ Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
+ target := filepath.Join(categoryDir, "input.wav")
+ if linkedName != "input.wav" {
+ Expect(os.WriteFile(target, []byte("input"), 0640)).To(Succeed())
+ }
+ preserved := filepath.Join(stagingDir, "ephemeral", "preserved-"+linkedName)
+ Expect(os.WriteFile(preserved, []byte("keep"), 0640)).To(Succeed())
+ Expect(os.Symlink(preserved, filepath.Join(categoryDir, linkedName))).To(Succeed())
+
+ stager, stop := startReleaseServer(stagingDir, "release-token")
+ Expect(stager.ReleaseRemote(context.Background(), "node-1", "ephemeral/request-id/audio/input.wav")).NotTo(Succeed(), linkedName)
+ stop()
+ Expect(preserved).To(BeAnExistingFile(), linkedName)
+ }
+ })
+
+ It("evicts the worker cache before deleting the shared object", func() {
+ storeRoot := GinkgoT().TempDir()
+ cacheRoot := GinkgoT().TempDir()
+ store, err := storage.NewFilesystemStore(storeRoot)
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, cacheRoot)
+ Expect(err).NotTo(HaveOccurred())
+ key := "ephemeral/request-id/audio/input.wav"
+ Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed())
+
+ client := &releaseTestMessaging{}
+ client.onRequest = func() {
+ exists, existsErr := store.Exists(context.Background(), key)
+ Expect(existsErr).NotTo(HaveOccurred())
+ Expect(exists).To(BeTrue())
+ }
+ stager := NewS3NATSFileStager(fm, client)
+ Expect(stager.ReleaseRemote(context.Background(), "node.one", key)).To(Succeed())
+
+ Expect(client.requestCalled).To(BeTrue())
+ Expect(client.subject).To(Equal(messaging.SubjectNodeFilesRelease("node.one")))
+ var payload fileReleaseRequest
+ Expect(json.Unmarshal(client.payload, &payload)).To(Succeed())
+ Expect(payload.Key).To(Equal(key))
+ exists, err := store.Exists(context.Background(), key)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(exists).To(BeFalse())
+ })
+
+ It("evicts a request's S3 inputs with one NATS round trip", func() {
+ storeRoot := GinkgoT().TempDir()
+ store, err := storage.NewFilesystemStore(storeRoot)
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ keys := []string{
+ "ephemeral/audio/request-id/input.wav",
+ "ephemeral/images/request-id/frame.jpg",
+ }
+ for _, key := range keys {
+ Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed())
+ }
+ client := &releaseTestMessaging{}
+ stager := NewS3NATSFileStager(fm, client)
+
+ Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed())
+
+ Expect(client.requestCount).To(Equal(1))
+ var payload fileReleaseRequest
+ Expect(json.Unmarshal(client.payload, &payload)).To(Succeed())
+ Expect(payload.Key).To(BeEmpty())
+ Expect(payload.RequestID).To(Equal("request-id"))
+ for _, key := range keys {
+ exists, existsErr := store.Exists(context.Background(), key)
+ Expect(existsErr).NotTo(HaveOccurred())
+ Expect(exists).To(BeFalse())
+ }
+ })
+
+ It("keeps worker coordination fixed-size for large requests", func() {
+ store, err := storage.NewFilesystemStore(GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ keys := make([]string, 2048)
+ for i := range keys {
+ keys[i] = fmt.Sprintf("ephemeral/inputs/request-id/input-%d.bin", i)
+ }
+ client := &releaseTestMessaging{}
+ stager := NewS3NATSFileStager(fm, client)
+
+ Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed())
+
+ Expect(client.requestCount).To(Equal(1))
+ Expect(len(client.payload)).To(BeNumerically("<", 128))
+ var payload fileReleaseRequest
+ Expect(json.Unmarshal(client.payload, &payload)).To(Succeed())
+ Expect(payload.RequestID).To(Equal("request-id"))
+ })
+
+ It("falls back to exact NATS releases for an older worker", func() {
+ store, err := storage.NewFilesystemStore(GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ keys := []string{
+ "ephemeral/audio/request-id/input.wav",
+ "ephemeral/images/request-id/frame.jpg",
+ }
+ for _, key := range keys {
+ Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed())
+ }
+ client := &releaseTestMessaging{replies: [][]byte{
+ []byte(`{"error":"batch payload unsupported"}`),
+ []byte(`{}`),
+ []byte(`{}`),
+ }}
+ stager := NewS3NATSFileStager(fm, client)
+
+ Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed())
+
+ Expect(client.requestCount).To(Equal(3))
+ for _, key := range keys {
+ exists, existsErr := store.Exists(context.Background(), key)
+ Expect(existsErr).NotTo(HaveOccurred())
+ Expect(exists).To(BeFalse())
+ }
+ })
+
+ It("rejects release batches that mix request IDs", func() {
+ keys := []string{
+ "ephemeral/audio/request-one/input.wav",
+ "ephemeral/images/request-two/frame.jpg",
+ }
+ Expect(validateEphemeralRequestRelease("request-one", keys)).To(MatchError(ContainSubstring("mixes request IDs")))
+ })
+
+ It("does not send a release request after cleanup is canceled", func() {
+ store, err := storage.NewFilesystemStore(GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ client := &releaseTestMessaging{}
+ stager := NewS3NATSFileStager(fm, client)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ Expect(stager.ReleaseRemote(ctx, "node.one", "ephemeral/request-id/audio/input.wav")).To(MatchError(context.Canceled))
+ Expect(client.requestCalled).To(BeFalse())
+ })
+
+ It("bounds the NATS release wait by the remaining cleanup deadline", func() {
+ store, err := storage.NewFilesystemStore(GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(store, GinkgoT().TempDir())
+ Expect(err).NotTo(HaveOccurred())
+ client := &releaseTestMessaging{}
+ stager := NewS3NATSFileStager(fm, client)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ Expect(stager.ReleaseRemote(ctx, "node.one", "ephemeral/request-id/audio/input.wav")).To(Succeed())
+ Expect(client.timeout).To(BeNumerically(">", time.Second))
+ Expect(client.timeout).To(BeNumerically("<=", 2*time.Second))
+ })
+})
diff --git a/core/services/nodes/file_stager_s3.go b/core/services/nodes/file_stager_s3.go
index 0d3847b7c..fff6d1859 100644
--- a/core/services/nodes/file_stager_s3.go
+++ b/core/services/nodes/file_stager_s3.go
@@ -2,6 +2,7 @@ package nodes
import (
"context"
+ "errors"
"fmt"
"time"
@@ -48,6 +49,15 @@ type fileStageReply struct {
Error string `json:"error,omitempty"`
}
+type fileReleaseRequest struct {
+ Key string `json:"key,omitempty"`
+ RequestID string `json:"request_id,omitempty"`
+}
+
+type fileReleaseReply struct {
+ Error string `json:"error,omitempty"`
+}
+
type fileTempRequest struct{}
type fileTempReply struct {
@@ -181,3 +191,79 @@ func (s *S3NATSFileStager) StageRemoteToStore(ctx context.Context, nodeID, remot
return nil
}
+
+// ReleaseRemote evicts one exact ephemeral key from the worker before deleting
+// the shared object.
+func (s *S3NATSFileStager) ReleaseRemote(ctx context.Context, nodeID, key string) error {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ return err
+ }
+ if err := s.releaseWorkerKeys(ctx, nodeID, fileReleaseRequest{Key: key}); err != nil {
+ return err
+ }
+ if err := s.fm.Delete(ctx, key); err != nil {
+ return fmt.Errorf("deleting shared object %q: %w", key, err)
+ }
+ return nil
+}
+
+// ReleaseRemoteRequest evicts one inference's inputs with one NATS round trip.
+// A worker that only understands the exact-key payload returns an error, so the
+// frontend retries each key during a rolling upgrade.
+func (s *S3NATSFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, requestID string, keys []string) error {
+ if err := validateEphemeralRequestRelease(requestID, keys); err != nil {
+ return err
+ }
+ if err := s.releaseWorkerKeys(ctx, nodeID, fileReleaseRequest{RequestID: requestID}); err != nil {
+ var fallbackErrors []error
+ for _, key := range keys {
+ if fallbackErr := s.ReleaseRemote(ctx, nodeID, key); fallbackErr != nil {
+ fallbackErrors = append(fallbackErrors, fallbackErr)
+ }
+ }
+ if fallbackErr := errors.Join(fallbackErrors...); fallbackErr != nil {
+ return errors.Join(err, fmt.Errorf("exact-key release fallback: %w", fallbackErr))
+ }
+ return nil
+ }
+ var deleteErrors []error
+ for _, key := range keys {
+ if err := s.fm.Delete(ctx, key); err != nil {
+ deleteErrors = append(deleteErrors, fmt.Errorf("deleting shared object %q: %w", key, err))
+ }
+ }
+ return errors.Join(deleteErrors...)
+}
+
+func (s *S3NATSFileStager) releaseWorkerKeys(ctx context.Context, nodeID string, request fileReleaseRequest) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ timeout := 30 * time.Second
+ if deadline, ok := ctx.Deadline(); ok {
+ remaining := time.Until(deadline)
+ if remaining <= 0 {
+ return context.DeadlineExceeded
+ }
+ timeout = min(timeout, remaining)
+ }
+ reply, err := messaging.RequestJSON[fileReleaseRequest, fileReleaseReply](
+ s.nats,
+ messaging.SubjectNodeFilesRelease(nodeID),
+ request,
+ timeout,
+ )
+ if err != nil {
+ if contextErr := ctx.Err(); contextErr != nil {
+ return contextErr
+ }
+ return err
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if reply.Error != "" {
+ return fmt.Errorf("backend release failed: %s", reply.Error)
+ }
+ return nil
+}
diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go
index bfc202c82..6c911780a 100644
--- a/core/services/nodes/file_staging_client.go
+++ b/core/services/nodes/file_staging_client.go
@@ -16,8 +16,11 @@ import (
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/xlog"
ggrpc "google.golang.org/grpc"
+ "google.golang.org/protobuf/proto"
)
+const stagedInputReleaseTimeout = 30 * time.Second
+
// FileStagingClient wraps a grpc.Backend to transparently handle file transfer
// for distributed mode. Input files are staged on the backend node before the
// gRPC call. Output files are retrieved from the backend after the call.
@@ -48,21 +51,70 @@ func NewFileStagingClient(inner grpc.Backend, stager FileStager, nodeID string)
// requestID generates a unique ID for ephemeral file keys.
func requestID() string {
- return uuid.New().String()[:8]
+ return uuid.NewString()
+}
+
+type stagedInputLifecycle struct {
+ client *FileStagingClient
+ requestID string
+ keys []string
+ seen map[string]struct{}
+}
+
+func (f *FileStagingClient) newStagedInputLifecycle() *stagedInputLifecycle {
+ return &stagedInputLifecycle{
+ client: f,
+ requestID: requestID(),
+ keys: []string{},
+ seen: map[string]struct{}{},
+ }
+}
+
+func (l *stagedInputLifecycle) track(key string) {
+ if _, ok := l.seen[key]; ok {
+ return
+ }
+ l.seen[key] = struct{}{}
+ l.keys = append(l.keys, key)
+}
+
+func (l *stagedInputLifecycle) release() {
+ if len(l.keys) == 0 {
+ return
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), stagedInputReleaseTimeout)
+ defer cancel()
+ if releaser, ok := l.client.stager.(RequestFileReleaser); ok {
+ if err := releaser.ReleaseRemoteRequest(ctx, l.client.nodeID, l.requestID, l.keys); err != nil {
+ xlog.Warn("Failed to release staged request inputs", "node", l.client.nodeID, "requestID", l.requestID, "keyCount", len(l.keys), "error", err)
+ }
+ return
+ }
+ for _, key := range l.keys {
+ if err := l.client.stager.ReleaseRemote(ctx, l.client.nodeID, key); err != nil {
+ xlog.Warn("Failed to release staged input", "node", l.client.nodeID, "key", key, "error", err)
+ }
+ }
}
// stageInputFile uploads a local file to the remote node via the FileStager.
-// Returns the remote-local path and the ephemeral key.
-func (f *FileStagingClient) stageInputFile(ctx context.Context, reqID, localPath, category string) (string, string, error) {
+func (f *FileStagingClient) stageInputFile(
+ ctx context.Context,
+ lifecycle *stagedInputLifecycle,
+ localPath,
+ category string,
+) (string, error) {
basename := filepath.Base(localPath)
- key := storage.EphemeralKey(reqID, category, basename)
+ key := storage.EphemeralKey(lifecycle.requestID, category, basename)
+ lifecycle.track(key)
remotePath, err := f.stager.EnsureRemote(ctx, f.nodeID, localPath, key)
if err != nil {
- return "", "", fmt.Errorf("staging input file: %w", err)
+ return "", fmt.Errorf("staging input file: %w", err)
}
- return remotePath, key, nil
+ return remotePath, nil
}
// retrieveOutputFile retrieves an output file from the backend to a local path.
@@ -100,23 +152,37 @@ func (f *FileStagingClient) translateModelPath(frontendPath string) string {
}
func (f *FileStagingClient) Predict(ctx context.Context, in *pb.PredictOptions, opts ...ggrpc.CallOption) (*pb.Reply, error) {
- reqID := requestID()
- in, _ = f.stageMultimodalInputs(ctx, reqID, in)
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.PredictOptions)
+ var err error
+ in, err = f.stageMultimodalInputs(ctx, lifecycle, in)
+ if err != nil {
+ return nil, err
+ }
return f.Backend.Predict(ctx, in, opts...)
}
func (f *FileStagingClient) PredictStream(ctx context.Context, in *pb.PredictOptions, fn func(reply *pb.Reply), opts ...ggrpc.CallOption) error {
- reqID := requestID()
- in, _ = f.stageMultimodalInputs(ctx, reqID, in)
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.PredictOptions)
+ var err error
+ in, err = f.stageMultimodalInputs(ctx, lifecycle, in)
+ if err != nil {
+ return err
+ }
return f.Backend.PredictStream(ctx, in, fn, opts...)
}
func (f *FileStagingClient) GenerateImage(ctx context.Context, in *pb.GenerateImageRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.GenerateImageRequest)
// Stage input source image if present
if in.Src != "" && isFilePath(in.Src) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Src, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Src, "inputs")
if err != nil {
return nil, fmt.Errorf("staging image src: %w", err)
}
@@ -126,7 +192,7 @@ func (f *FileStagingClient) GenerateImage(ctx context.Context, in *pb.GenerateIm
// Stage reference images
for i, img := range in.RefImages {
if isFilePath(img) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, img, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, img, "inputs")
if err != nil {
return nil, fmt.Errorf("staging ref image: %w", err)
}
@@ -160,25 +226,27 @@ func (f *FileStagingClient) GenerateImage(ctx context.Context, in *pb.GenerateIm
}
func (f *FileStagingClient) GenerateVideo(ctx context.Context, in *pb.GenerateVideoRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.GenerateVideoRequest)
// Stage start/end images and optional audio conditioning.
if in.StartImage != "" && isFilePath(in.StartImage) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.StartImage, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.StartImage, "inputs")
if err != nil {
return nil, fmt.Errorf("staging start image: %w", err)
}
in.StartImage = backendPath
}
if in.EndImage != "" && isFilePath(in.EndImage) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.EndImage, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.EndImage, "inputs")
if err != nil {
return nil, fmt.Errorf("staging end image: %w", err)
}
in.EndImage = backendPath
}
if in.Audio != "" && isFilePath(in.Audio) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Audio, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Audio, "inputs")
if err != nil {
return nil, fmt.Errorf("staging video audio: %w", err)
}
@@ -210,11 +278,13 @@ func (f *FileStagingClient) GenerateVideo(ctx context.Context, in *pb.GenerateVi
}
func (f *FileStagingClient) Generate3D(ctx context.Context, in *pb.Generate3DRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.Generate3DRequest)
// Stage the conditioning image or existing GLB used by 3D post-processing.
if in.Src != "" && isFilePath(in.Src) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Src, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Src, "inputs")
if err != nil {
return nil, fmt.Errorf("staging 3D input asset: %w", err)
}
@@ -246,7 +316,9 @@ func (f *FileStagingClient) Generate3D(ctx context.Context, in *pb.Generate3DReq
}
func (f *FileStagingClient) TTS(ctx context.Context, in *pb.TTSRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.TTSRequest)
// Translate model path from frontend to remote worker path.
// The model and its companion files (e.g. .onnx.json) were already staged
@@ -257,7 +329,7 @@ func (f *FileStagingClient) TTS(ctx context.Context, in *pb.TTSRequest, opts ...
// Voice may be a named backend speaker or a request-scoped reference WAV.
// Only path-shaped values are staged; speaker IDs pass through unchanged.
if in.Voice != "" && isFilePath(in.Voice) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Voice, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Voice, "inputs")
if err != nil {
return nil, fmt.Errorf("staging TTS voice reference: %w", err)
}
@@ -289,14 +361,16 @@ func (f *FileStagingClient) TTS(ctx context.Context, in *pb.TTSRequest, opts ...
}
func (f *FileStagingClient) TTSStream(ctx context.Context, in *pb.TTSRequest, fn func(*pb.Reply), opts ...ggrpc.CallOption) error {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.TTSRequest)
// Translate model path from frontend to remote worker path (same as TTS above)
if in.Model != "" && isFilePath(in.Model) {
in.Model = f.translateModelPath(in.Model)
}
if in.Voice != "" && isFilePath(in.Voice) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Voice, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Voice, "inputs")
if err != nil {
return fmt.Errorf("staging streaming TTS voice reference: %w", err)
}
@@ -307,11 +381,13 @@ func (f *FileStagingClient) TTSStream(ctx context.Context, in *pb.TTSRequest, fn
}
func (f *FileStagingClient) SoundGeneration(ctx context.Context, in *pb.SoundGenerationRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.SoundGenerationRequest)
// Stage input source
if in.Src != nil && *in.Src != "" && isFilePath(*in.Src) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, *in.Src, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, *in.Src, "inputs")
if err != nil {
return nil, fmt.Errorf("staging sound src: %w", err)
}
@@ -342,12 +418,28 @@ func (f *FileStagingClient) SoundGeneration(ctx context.Context, in *pb.SoundGen
return result, nil
}
+func (f *FileStagingClient) SoundDetection(ctx context.Context, in *pb.SoundDetectionRequest, opts ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) {
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.SoundDetectionRequest)
+ if in.Src != "" && isFilePath(in.Src) {
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Src, "inputs")
+ if err != nil {
+ return nil, fmt.Errorf("staging audio for sound detection: %w", err)
+ }
+ in.Src = backendPath
+ }
+ return f.Backend.SoundDetection(ctx, in, opts...)
+}
+
func (f *FileStagingClient) AudioTranscription(ctx context.Context, in *pb.TranscriptRequest, opts ...ggrpc.CallOption) (*pb.TranscriptResult, error) {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.TranscriptRequest)
// Stage input audio file
if in.Dst != "" && isFilePath(in.Dst) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Dst, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Dst, "inputs")
if err != nil {
return nil, fmt.Errorf("staging audio for transcription: %w", err)
}
@@ -358,11 +450,13 @@ func (f *FileStagingClient) AudioTranscription(ctx context.Context, in *pb.Trans
}
func (f *FileStagingClient) AudioTranscriptionStream(ctx context.Context, in *pb.TranscriptRequest, fn func(chunk *pb.TranscriptStreamResponse), opts ...ggrpc.CallOption) error {
- reqID := requestID()
+ lifecycle := f.newStagedInputLifecycle()
+ defer lifecycle.release()
+ in = proto.Clone(in).(*pb.TranscriptRequest)
// Stage input audio file
if in.Dst != "" && isFilePath(in.Dst) {
- backendPath, _, err := f.stageInputFile(ctx, reqID, in.Dst, "inputs")
+ backendPath, err := f.stageInputFile(ctx, lifecycle, in.Dst, "inputs")
if err != nil {
return fmt.Errorf("staging audio for transcription stream: %w", err)
}
@@ -444,31 +538,46 @@ func (f *FileStagingClient) QuantizationProgress(ctx context.Context, in *pb.Qua
// stageMultimodalInputs stages Images, Videos, Audios fields in PredictOptions
// if they are file paths (not base64 or URLs).
-func (f *FileStagingClient) stageMultimodalInputs(ctx context.Context, reqID string, in *pb.PredictOptions) (*pb.PredictOptions, []string) {
- var keys []string
- in.Images = f.stagePathSlice(ctx, reqID, in.Images, "inputs", &keys)
- in.Videos = f.stagePathSlice(ctx, reqID, in.Videos, "inputs", &keys)
- in.Audios = f.stagePathSlice(ctx, reqID, in.Audios, "inputs", &keys)
- return in, keys
+func (f *FileStagingClient) stageMultimodalInputs(
+ ctx context.Context,
+ lifecycle *stagedInputLifecycle,
+ in *pb.PredictOptions,
+) (*pb.PredictOptions, error) {
+ var err error
+ in.Images, err = f.stagePathSlice(ctx, lifecycle, in.Images, "inputs")
+ if err != nil {
+ return nil, fmt.Errorf("staging predict images: %w", err)
+ }
+ in.Videos, err = f.stagePathSlice(ctx, lifecycle, in.Videos, "inputs")
+ if err != nil {
+ return nil, fmt.Errorf("staging predict videos: %w", err)
+ }
+ in.Audios, err = f.stagePathSlice(ctx, lifecycle, in.Audios, "inputs")
+ if err != nil {
+ return nil, fmt.Errorf("staging predict audios: %w", err)
+ }
+ return in, nil
}
-func (f *FileStagingClient) stagePathSlice(ctx context.Context, reqID string, paths []string, category string, keys *[]string) []string {
+func (f *FileStagingClient) stagePathSlice(
+ ctx context.Context,
+ lifecycle *stagedInputLifecycle,
+ paths []string,
+ category string,
+) ([]string, error) {
result := make([]string, len(paths))
for i, p := range paths {
if isFilePath(p) {
- backendPath, key, err := f.stageInputFile(ctx, reqID, p, category)
+ backendPath, err := f.stageInputFile(ctx, lifecycle, p, category)
if err != nil {
- xlog.Warn("Failed to stage multimodal file, passing through", "path", p, "error", err)
- result[i] = p
- continue
+ return nil, fmt.Errorf("staging %q: %w", p, err)
}
result[i] = backendPath
- *keys = append(*keys, key)
} else {
result[i] = p
}
}
- return result
+ return result, nil
}
// isFilePath checks if a string looks like a local file path (not base64 or URL).
diff --git a/core/services/nodes/file_staging_lifecycle_test.go b/core/services/nodes/file_staging_lifecycle_test.go
new file mode 100644
index 000000000..b8093dad6
--- /dev/null
+++ b/core/services/nodes/file_staging_lifecycle_test.go
@@ -0,0 +1,341 @@
+package nodes
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ grpc "github.com/mudler/LocalAI/pkg/grpc"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ ggrpc "google.golang.org/grpc"
+ "google.golang.org/protobuf/proto"
+)
+
+const fullUUIDPattern = `[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}`
+
+type lifecycleStager struct {
+ fakeFileStager
+ ensureErr error
+ ensureErrAt int
+ releaseErr error
+ releasedKeys []string
+ releaseBatches [][]string
+ releaseCtxErr []error
+ releaseHasDeadline []bool
+ releaseDeadlines []time.Time
+}
+
+func (s *lifecycleStager) EnsureRemote(ctx context.Context, nodeID, localPath, key string) (string, error) {
+ s.fakeFileStager.EnsureRemote(ctx, nodeID, localPath, key)
+ if s.ensureErr != nil && (s.ensureErrAt == 0 || len(s.ensureCalls) == s.ensureErrAt) {
+ return "", s.ensureErr
+ }
+ return "/remote/" + key, nil
+}
+
+func (s *lifecycleStager) ReleaseRemote(ctx context.Context, _ string, key string) error {
+ s.releasedKeys = append(s.releasedKeys, key)
+ s.releaseCtxErr = append(s.releaseCtxErr, ctx.Err())
+ deadline, ok := ctx.Deadline()
+ s.releaseHasDeadline = append(s.releaseHasDeadline, ok)
+ s.releaseDeadlines = append(s.releaseDeadlines, deadline)
+ return s.releaseErr
+}
+
+func (s *lifecycleStager) ReleaseRemoteRequest(ctx context.Context, _, _ string, keys []string) error {
+ s.releaseBatches = append(s.releaseBatches, append([]string(nil), keys...))
+ s.releasedKeys = append(s.releasedKeys, keys...)
+ s.releaseCtxErr = append(s.releaseCtxErr, ctx.Err())
+ deadline, ok := ctx.Deadline()
+ s.releaseHasDeadline = append(s.releaseHasDeadline, ok)
+ s.releaseDeadlines = append(s.releaseDeadlines, deadline)
+ return s.releaseErr
+}
+
+type lifecycleBackend struct {
+ grpc.Backend
+ predictResult *pb.Reply
+ predictErr error
+ predictCalls int
+ predictInput *pb.PredictOptions
+ streamCalls int
+ streamBlock <-chan struct{}
+ streamStarted chan<- struct{}
+}
+
+func (b *lifecycleBackend) Predict(_ context.Context, in *pb.PredictOptions, _ ...ggrpc.CallOption) (*pb.Reply, error) {
+ b.predictCalls++
+ b.predictInput = proto.Clone(in).(*pb.PredictOptions)
+ if b.predictResult == nil {
+ b.predictResult = &pb.Reply{}
+ }
+ return b.predictResult, b.predictErr
+}
+
+func (b *lifecycleBackend) PredictStream(_ context.Context, _ *pb.PredictOptions, _ func(*pb.Reply), _ ...ggrpc.CallOption) error {
+ b.streamCalls++
+ if b.streamStarted != nil {
+ b.streamStarted <- struct{}{}
+ }
+ if b.streamBlock != nil {
+ <-b.streamBlock
+ }
+ return nil
+}
+
+func (b *lifecycleBackend) GenerateImage(_ context.Context, _ *pb.GenerateImageRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ return &pb.Result{Success: true}, nil
+}
+
+func (b *lifecycleBackend) GenerateVideo(_ context.Context, _ *pb.GenerateVideoRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ return &pb.Result{Success: true}, nil
+}
+
+func (b *lifecycleBackend) Generate3D(_ context.Context, _ *pb.Generate3DRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ return &pb.Result{Success: true}, nil
+}
+
+func (b *lifecycleBackend) TTS(_ context.Context, _ *pb.TTSRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ return &pb.Result{Success: true}, nil
+}
+
+func (b *lifecycleBackend) TTSStream(_ context.Context, _ *pb.TTSRequest, _ func(*pb.Reply), _ ...ggrpc.CallOption) error {
+ return nil
+}
+
+func (b *lifecycleBackend) SoundGeneration(_ context.Context, _ *pb.SoundGenerationRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
+ return &pb.Result{Success: true}, nil
+}
+
+func (b *lifecycleBackend) SoundDetection(_ context.Context, _ *pb.SoundDetectionRequest, _ ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) {
+ return &pb.SoundDetectionResponse{}, nil
+}
+
+func (b *lifecycleBackend) AudioTranscription(_ context.Context, _ *pb.TranscriptRequest, _ ...ggrpc.CallOption) (*pb.TranscriptResult, error) {
+ return &pb.TranscriptResult{}, nil
+}
+
+func (b *lifecycleBackend) AudioTranscriptionStream(_ context.Context, _ *pb.TranscriptRequest, _ func(*pb.TranscriptStreamResponse), _ ...ggrpc.CallOption) error {
+ return nil
+}
+
+var _ = Describe("FileStagingClient request lifecycle", func() {
+ It("uses a full UUID for ephemeral request keys", func() {
+ Expect(requestID()).To(MatchRegexp(`^` + fullUUIDPattern + `$`))
+ })
+
+ It("releases every staged key and preserves caller requests", func(ctx SpecContext) {
+ tests := []struct {
+ name string
+ keyCount int
+ invoke func(*FileStagingClient) proto.Message
+ }{
+ {name: "predict", keyCount: 3, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.PredictOptions{Images: []string{"/tmp/image.png"}, Videos: []string{"/tmp/video.mp4"}, Audios: []string{"/tmp/audio.wav"}}
+ original := proto.Clone(request)
+ _, err := client.Predict(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "predict stream", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.PredictOptions{Images: []string{"/tmp/image.png"}}
+ original := proto.Clone(request)
+ Expect(client.PredictStream(ctx, request, func(*pb.Reply) {})).To(Succeed())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "image generation", keyCount: 2, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.GenerateImageRequest{Src: "/tmp/source.png", RefImages: []string{"/tmp/reference.png"}}
+ original := proto.Clone(request)
+ _, err := client.GenerateImage(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "video generation", keyCount: 3, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.GenerateVideoRequest{StartImage: "/tmp/start.png", EndImage: "/tmp/end.png", Audio: "/tmp/audio.wav"}
+ original := proto.Clone(request)
+ _, err := client.GenerateVideo(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "3D generation", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.Generate3DRequest{Src: "/tmp/source.glb"}
+ original := proto.Clone(request)
+ _, err := client.Generate3D(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "TTS", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.TTSRequest{Voice: "/tmp/voice.wav"}
+ original := proto.Clone(request)
+ _, err := client.TTS(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "streaming TTS", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.TTSRequest{Voice: "/tmp/voice.wav"}
+ original := proto.Clone(request)
+ Expect(client.TTSStream(ctx, request, func(*pb.Reply) {})).To(Succeed())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "sound generation", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ source := "/tmp/source.wav"
+ request := &pb.SoundGenerationRequest{Src: &source}
+ original := proto.Clone(request)
+ _, err := client.SoundGeneration(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "sound detection", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.SoundDetectionRequest{Src: "/tmp/source.wav"}
+ original := proto.Clone(request)
+ _, err := client.SoundDetection(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "transcription", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.TranscriptRequest{Dst: "/tmp/source.wav"}
+ original := proto.Clone(request)
+ _, err := client.AudioTranscription(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ {name: "streaming transcription", keyCount: 1, invoke: func(client *FileStagingClient) proto.Message {
+ request := &pb.TranscriptRequest{Dst: "/tmp/source.wav"}
+ original := proto.Clone(request)
+ Expect(client.AudioTranscriptionStream(ctx, request, func(*pb.TranscriptStreamResponse) {})).To(Succeed())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ return request
+ }},
+ }
+
+ for _, test := range tests {
+ By(test.name)
+ stager := &lifecycleStager{}
+ client := NewFileStagingClient(&lifecycleBackend{}, stager, "worker-1")
+ test.invoke(client)
+ Expect(stager.ensureCalls).To(HaveLen(test.keyCount))
+ Expect(stager.releasedKeys).To(Equal(keysFromEnsureCalls(stager.ensureCalls)))
+ Expect(stager.releaseBatches).To(Equal([][]string{keysFromEnsureCalls(stager.ensureCalls)}))
+ Expect(stager.releaseCtxErr).To(Equal([]error{nil}))
+ Expect(stager.releaseHasDeadline).To(HaveLen(1))
+ for _, hasDeadline := range stager.releaseHasDeadline {
+ Expect(hasDeadline).To(BeTrue())
+ }
+ for _, deadline := range stager.releaseDeadlines {
+ Expect(time.Until(deadline)).To(BeNumerically(">", 0))
+ Expect(time.Until(deadline)).To(BeNumerically("<=", time.Minute))
+ }
+ }
+ })
+
+ It("tracks a key before staging so a partial upload failure is released", func(ctx SpecContext) {
+ uploadErr := errors.New("upload failed")
+ stager := &lifecycleStager{ensureErr: uploadErr, releaseErr: errors.New("release failed")}
+ client := NewFileStagingClient(&lifecycleBackend{}, stager, "worker-1")
+
+ _, err := client.GenerateImage(ctx, &pb.GenerateImageRequest{Src: "/tmp/source.png"})
+
+ Expect(err).To(MatchError(ContainSubstring("upload failed")))
+ Expect(stager.ensureCalls).To(HaveLen(1))
+ Expect(stager.releasedKeys).To(Equal(keysFromEnsureCalls(stager.ensureCalls)))
+ })
+
+ It("does not invoke predict when multimodal staging fails", func(ctx SpecContext) {
+ uploadErr := errors.New("ephemeral capacity exceeded")
+ stager := &lifecycleStager{ensureErr: uploadErr, ensureErrAt: 2}
+ backend := &lifecycleBackend{}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+ request := &pb.PredictOptions{Images: []string{"/tmp/first.png", "/tmp/second.png"}}
+ original := proto.Clone(request)
+
+ result, err := client.Predict(ctx, request)
+
+ Expect(result).To(BeNil())
+ Expect(err).To(MatchError(ContainSubstring("ephemeral capacity exceeded")))
+ Expect(backend.predictCalls).To(BeZero())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ Expect(stager.ensureCalls).To(HaveLen(2))
+ Expect(stager.releasedKeys).To(Equal(keysFromEnsureCalls(stager.ensureCalls)))
+ })
+
+ It("does not invoke streaming predict when multimodal staging fails", func(ctx SpecContext) {
+ uploadErr := errors.New("ephemeral capacity exceeded")
+ stager := &lifecycleStager{ensureErr: uploadErr}
+ backend := &lifecycleBackend{}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+ request := &pb.PredictOptions{Audios: []string{"/tmp/audio.wav"}}
+ original := proto.Clone(request)
+
+ err := client.PredictStream(ctx, request, func(*pb.Reply) {})
+
+ Expect(err).To(MatchError(ContainSubstring("ephemeral capacity exceeded")))
+ Expect(backend.streamCalls).To(BeZero())
+ Expect(proto.Equal(request, original)).To(BeTrue())
+ Expect(stager.releasedKeys).To(Equal(keysFromEnsureCalls(stager.ensureCalls)))
+ })
+
+ It("uses an active bounded cleanup context after caller cancellation", func() {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ stager := &lifecycleStager{}
+ client := NewFileStagingClient(&lifecycleBackend{}, stager, "worker-1")
+
+ _, err := client.Predict(ctx, &pb.PredictOptions{Images: []string{"/tmp/image.png"}})
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(stager.releaseCtxErr).To(Equal([]error{nil}))
+ })
+
+ It("does not release streaming inputs before the backend completes", func(ctx SpecContext) {
+ block := make(chan struct{})
+ started := make(chan struct{}, 1)
+ backend := &lifecycleBackend{streamBlock: block, streamStarted: started}
+ stager := &lifecycleStager{}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+ done := make(chan error, 1)
+
+ go func() {
+ done <- client.PredictStream(ctx, &pb.PredictOptions{Images: []string{"/tmp/image.png"}}, func(*pb.Reply) {})
+ }()
+
+ Eventually(started).Should(Receive())
+ Expect(stager.ensureCalls).To(HaveLen(1))
+ Expect(stager.releasedKeys).To(BeEmpty())
+ close(block)
+ Eventually(done).Should(Receive(Succeed()))
+ Expect(stager.releasedKeys).To(Equal(keysFromEnsureCalls(stager.ensureCalls)))
+ })
+
+ It("does not replace a backend result when cleanup fails", func(ctx SpecContext) {
+ reply := &pb.Reply{Message: []byte("ok")}
+ backend := &lifecycleBackend{predictResult: reply}
+ stager := &lifecycleStager{releaseErr: errors.New("release failed")}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+
+ result, err := client.Predict(ctx, &pb.PredictOptions{Images: []string{"/tmp/image.png"}})
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(result).To(BeIdenticalTo(reply))
+ })
+})
+
+func keysFromEnsureCalls(calls []ensureCall) []string {
+ keys := make([]string, len(calls))
+ for i, call := range calls {
+ keys[i] = call.key
+ }
+ return keys
+}
diff --git a/core/services/nodes/file_staging_sound_detection_test.go b/core/services/nodes/file_staging_sound_detection_test.go
new file mode 100644
index 000000000..2fc6fb7e3
--- /dev/null
+++ b/core/services/nodes/file_staging_sound_detection_test.go
@@ -0,0 +1,98 @@
+package nodes
+
+import (
+ "context"
+ "errors"
+
+ grpc "github.com/mudler/LocalAI/pkg/grpc"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ ggrpc "google.golang.org/grpc"
+)
+
+type soundStagingBackend struct {
+ grpc.Backend
+ request *pb.SoundDetectionRequest
+ err error
+}
+
+func (b *soundStagingBackend) SoundDetection(_ context.Context, in *pb.SoundDetectionRequest, _ ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) {
+ b.request = in
+ return &pb.SoundDetectionResponse{}, b.err
+}
+
+type soundStagingFailure struct{ FileStager }
+
+func (s *soundStagingFailure) EnsureRemote(context.Context, string, string, string) (string, error) {
+ return "", errors.New("upload failed")
+}
+
+func (s *soundStagingFailure) ReleaseRemote(context.Context, string, string) error {
+ return nil
+}
+
+type soundRouteFactory struct{ client grpc.Backend }
+
+func (f *soundRouteFactory) NewClient(string, bool) grpc.Backend { return f.client }
+
+var _ = Describe("FileStagingClient sound detection", func() {
+ It("stages sound audio through the client returned by SmartRouter.Route", func(ctx SpecContext) {
+ node := &BackendNode{ID: "worker-1", Name: "worker", Address: "10.0.0.1:50051"}
+ reg := &fakeModelRouter{
+ findAndLockNode: node,
+ findAndLockNM: &NodeModel{NodeID: node.ID, ModelName: "ced", Address: "10.0.0.1:9001"},
+ }
+ backend := &soundStagingBackend{Backend: &stubBackend{healthResult: true}}
+ stager := &fakeFileStager{}
+ router := NewSmartRouter(reg, SmartRouterOptions{
+ ClientFactory: &soundRouteFactory{client: backend},
+ FileStager: stager,
+ Unloader: &fakeUnloader{},
+ })
+ result, err := router.Route(ctx, "ced", "ced.gguf", "ced", "", nil, false)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(result).NotTo(BeNil())
+ defer result.Release()
+ request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf"}
+ _, err = result.Client.SoundDetection(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(stager.ensureCalls).To(HaveLen(1))
+ Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src))
+ Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key))
+ Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav"))
+ })
+
+ It("stages audio on the worker without changing the caller's request", func(ctx SpecContext) {
+ backend := &soundStagingBackend{}
+ stager := &fakeFileStager{}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+ request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf", TopK: 5, Threshold: 0.25}
+ _, err := client.SoundDetection(ctx, request)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(stager.ensureCalls).To(HaveLen(1))
+ Expect(stager.ensureCalls[0].nodeID).To(Equal("worker-1"))
+ Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src))
+ Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key))
+ Expect(backend.request.ModelIdentity).To(Equal(request.ModelIdentity))
+ Expect(backend.request.TopK).To(Equal(request.TopK))
+ Expect(backend.request.Threshold).To(Equal(request.Threshold))
+ Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav"))
+ })
+ It("does not call the backend when staging fails", func(ctx SpecContext) {
+ backend := &soundStagingBackend{}
+ client := NewFileStagingClient(backend, &soundStagingFailure{}, "worker-1")
+ _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: "/tmp/clip.wav"})
+ Expect(err).To(MatchError(ContainSubstring("upload failed")))
+ Expect(backend.request).To(BeNil())
+ })
+ It("passes through requests without a file and preserves backend errors", func(ctx SpecContext) {
+ failure := errors.New("classifier failed")
+ backend := &soundStagingBackend{err: failure}
+ stager := &fakeFileStager{}
+ client := NewFileStagingClient(backend, stager, "worker-1")
+ _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{})
+ Expect(err).To(MatchError(failure))
+ Expect(stager.ensureCalls).To(BeEmpty())
+ })
+})
diff --git a/core/services/nodes/file_staging_tts_test.go b/core/services/nodes/file_staging_tts_test.go
index a4e4f57f4..f9ff2a5d7 100644
--- a/core/services/nodes/file_staging_tts_test.go
+++ b/core/services/nodes/file_staging_tts_test.go
@@ -39,7 +39,8 @@ var _ = Describe("FileStagingClient TTS references", func() {
Expect(stager.ensureCalls).To(HaveLen(1))
Expect(stager.ensureCalls[0].localPath).To(Equal("/data/voice-profiles/profile/reference.wav"))
Expect(backend.ttsRequest.Voice).To(HavePrefix("/remote/ephemeral/"))
- Expect(backend.ttsRequest.Voice).To(MatchRegexp(`/inputs/[0-9a-f]{8}/reference\.wav$`))
+ voicePathPattern := `/inputs/` + fullUUIDPattern + `/reference\.wav$`
+ Expect(backend.ttsRequest.Voice).To(MatchRegexp(voicePathPattern))
})
It("stages a reference WAV before streaming synthesis", func(ctx SpecContext) {
diff --git a/core/services/nodes/file_transfer_finalize_test.go b/core/services/nodes/file_transfer_finalize_test.go
new file mode 100644
index 000000000..bc0753b0e
--- /dev/null
+++ b/core/services/nodes/file_transfer_finalize_test.go
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: MIT
+
+package nodes
+
+import (
+ "context"
+ "crypto/sha256"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Recovering unfinished file finalization", func() {
+ It("stages a full-size unfinished upload and subsequently skips the committed file", func() {
+ dir := GinkgoT().TempDir()
+ content := []byte("complete model bytes")
+ hash := fmt.Sprintf("%x", sha256.Sum256(content))
+ remote := filepath.Join(dir, "model.bin")
+ Expect(os.WriteFile(remote, content, 0600)).To(Succeed())
+ Expect(os.WriteFile(remote+targetSidecarSuffix, []byte(hash), 0600)).To(Succeed())
+ local := filepath.Join(GinkgoT().TempDir(), "model.bin")
+ Expect(os.WriteFile(local, content, 0600)).To(Succeed())
+
+ puts := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodHead:
+ handleHead(w, r, dir, "", "", "model.bin")
+ case http.MethodPut:
+ puts++
+ handleUpload(w, r, dir, "", "", "model.bin", 0)
+ }
+ }))
+ DeferCleanup(server.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(server.URL, "http://"), nil
+ }, "")
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ result, err := stager.EnsureRemote(ctx, "worker", local, "model.bin")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(result).To(Equal(remote))
+ Expect(remote + targetSidecarSuffix).NotTo(BeAnExistingFile())
+ Expect(os.ReadFile(remote + hashSidecarSuffix)).To(Equal([]byte(hash)))
+ Expect(os.ReadFile(remote)).To(Equal(content))
+ result, err = stager.EnsureRemote(ctx, "worker", local, "model.bin")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(result).To(Equal(remote))
+ Expect(puts).To(Equal(1))
+ })
+
+ DescribeTable("validates existing bytes before acknowledging a retry",
+ func(content string, expectedStatus int) {
+ dir := GinkgoT().TempDir()
+ remote := filepath.Join(dir, "model.bin")
+ hash := fmt.Sprintf("%x", sha256.Sum256([]byte("good")))
+ Expect(os.WriteFile(remote, []byte(content), 0600)).To(Succeed())
+ Expect(os.WriteFile(remote+targetSidecarSuffix, []byte(hash), 0600)).To(Succeed())
+ // A cached hash must not conceal corrupt bytes left by an interrupted upload.
+ Expect(os.WriteFile(remote+hashSidecarSuffix, []byte(hash), 0600)).To(Succeed())
+ req := httptest.NewRequest(http.MethodPut, "/v1/files/model.bin", strings.NewReader("good"))
+ req.Header.Set("Content-Range", "bytes 0-3/4")
+ req.Header.Set(HeaderContentSHA256, hash)
+ response := httptest.NewRecorder()
+ handleUpload(response, req, dir, "", "", "model.bin", 0)
+ Expect(response.Code).To(Equal(expectedStatus))
+ if expectedStatus == http.StatusBadRequest {
+ Expect(response.Body.String()).To(ContainSubstring("sha256 mismatch"))
+ Expect(remote).NotTo(BeAnExistingFile())
+ Expect(remote + targetSidecarSuffix).NotTo(BeAnExistingFile())
+ Expect(remote + hashSidecarSuffix).NotTo(BeAnExistingFile())
+ } else {
+ Expect(os.ReadFile(remote)).To(Equal([]byte(content)))
+ Expect(remote + targetSidecarSuffix).To(BeAnExistingFile())
+ }
+ },
+ Entry("rejects corrupt full-size content", "evil", http.StatusBadRequest),
+ Entry("preserves partial content for resume", "go", http.StatusRequestedRangeNotSatisfiable),
+ Entry("does not finalize oversized content", "good-extra", http.StatusRequestedRangeNotSatisfiable),
+ )
+})
diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go
index 0fc5ac634..486753b77 100644
--- a/core/services/nodes/file_transfer_server.go
+++ b/core/services/nodes/file_transfer_server.go
@@ -6,6 +6,7 @@ import (
"crypto/subtle"
"encoding/hex"
"encoding/json"
+ "errors"
"fmt"
"io"
"net"
@@ -22,6 +23,7 @@ import (
"github.com/mudler/LocalAI/core/services/storage"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/model"
+ "github.com/mudler/LocalAI/pkg/safefile"
"github.com/mudler/xlog"
)
@@ -32,8 +34,9 @@ const (
HeaderFileSize = "X-File-Size"
// HeaderTargetSHA256 is set on HEAD responses for partial (resumable) uploads
// to expose the expected final SHA-256 of the in-progress file. When set,
- // the file on disk is not yet the full content — the client may resume by
- // PUT'ing the remainder with a matching X-Content-SHA256 header.
+ // the file on disk is not yet verified — the client may resume by
+ // PUT'ing the remainder with a matching X-Content-SHA256 header. A full-size
+ // file can still carry this marker if finalization was interrupted.
HeaderTargetSHA256 = "X-Target-SHA256"
hashSidecarSuffix = ".sha256"
// targetSidecarSuffix stores the expected final SHA-256 of a partially
@@ -48,17 +51,23 @@ const (
// Auth is via Bearer token (registration token), using constant-time comparison.
// A nil readiness fails open, keeping /readyz's historical always-200 answer.
func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) {
+ return StartFileTransferServerWithCapacity(addr, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, logStore...)
+}
+
+// StartFileTransferServerWithCapacity starts the file transfer server with a
+// worker-local guard for per-request ephemeral inputs.
+func StartFileTransferServerWithCapacity(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, logStore ...*model.BackendLogStore) (*http.Server, error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("listen %s: %w", addr, err)
}
- return StartFileTransferServerWithReadiness(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, logStore...)
+ return startFileTransferServer(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, capacity, logStore...)
}
// StartFileTransferServerWithListener starts the server on an existing listener.
// This avoids the TOCTOU race of closing a listener and re-binding to the same port.
func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, logStore ...*model.BackendLogStore) (*http.Server, error) {
- return StartFileTransferServerWithReadiness(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, nil, logStore...)
+ return startFileTransferServer(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, nil, nil, logStore...)
}
// StartFileTransferServerWithReadiness is StartFileTransferServerWithListener
@@ -66,6 +75,10 @@ func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir
// the probe keeps its historical always-200 behaviour for callers that have no
// meaningful readiness signal to report.
func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) {
+ return startFileTransferServer(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, logStore...)
+}
+
+func startFileTransferServer(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, logStore ...*model.BackendLogStore) (*http.Server, error) {
if err := os.MkdirAll(stagingDir, 0750); err != nil {
return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err)
}
@@ -100,6 +113,18 @@ func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDi
handleListDir(w, r, stagingDir, modelsDir, dataDir, key)
})
+ mux.HandleFunc("/v1/files-release", func(w http.ResponseWriter, r *http.Request) {
+ if !checkBearerToken(r, token) {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ if r.Method != http.MethodDelete {
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ handleReleaseBatchWithCapacity(w, r, stagingDir, capacity)
+ })
+
mux.HandleFunc("/v1/files/", func(w http.ResponseWriter, r *http.Request) {
if !checkBearerToken(r, token) {
xlog.Debug("HTTP file transfer: unauthorized request", "method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr)
@@ -115,12 +140,16 @@ func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDi
case http.MethodHead:
handleHead(w, r, stagingDir, modelsDir, dataDir, key)
case http.MethodPut:
- handleUpload(w, r, stagingDir, modelsDir, dataDir, key, maxUploadSize)
+ handleUploadWithCapacity(w, r, stagingDir, modelsDir, dataDir, key, maxUploadSize, capacity)
case http.MethodGet:
handleDownload(w, r, stagingDir, modelsDir, dataDir, key)
+ case http.MethodDelete:
+ handleReleaseWithCapacity(w, r, stagingDir, key, capacity)
case http.MethodPost:
if key == "temp" {
handleAllocTemp(w, r, stagingDir)
+ } else if r.URL.Query().Get("claim") == "1" {
+ handleClaimWithCapacity(w, r, stagingDir, key, capacity)
} else {
http.Error(w, "not found", http.StatusNotFound)
}
@@ -181,6 +210,163 @@ func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDi
return server, nil
}
+// handleClaimWithCapacity marks an existing ephemeral file as owned by the
+// request that just verified its content.
+func handleClaimWithCapacity(w http.ResponseWriter, _ *http.Request, stagingDir, key string, capacity EphemeralCapacity) {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if operations, ok := capacity.(ephemeralRequestOperationCapacity); ok {
+ requestID := strings.Split(key, "/")[2]
+ if err := operations.BeginRequestOperation(requestID); err != nil {
+ http.Error(w, err.Error(), http.StatusConflict)
+ return
+ }
+ defer operations.EndRequestOperation(requestID)
+ }
+ filePath := filepath.Join(stagingDir, filepath.FromSlash(key))
+ if err := validatePathInDir(filePath, stagingDir); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ info, err := os.Lstat(filePath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ http.Error(w, "not found", http.StatusNotFound)
+ } else {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ return
+ }
+ if !info.Mode().IsRegular() {
+ http.Error(w, "ephemeral path is not a regular file", http.StatusBadRequest)
+ return
+ }
+ if capacity != nil {
+ if err := capacity.Claim(filePath); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ http.Error(w, "not found", http.StatusNotFound)
+ return
+ }
+ http.Error(w, err.Error(), http.StatusInsufficientStorage)
+ return
+ }
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
+
+func handleRelease(w http.ResponseWriter, _ *http.Request, stagingDir, key string) {
+ handleReleaseWithCapacity(w, nil, stagingDir, key, nil)
+}
+
+func handleReleaseWithCapacity(w http.ResponseWriter, _ *http.Request, stagingDir, key string, capacity EphemeralCapacity) {
+ if err := releaseEphemeralStagingKey(stagingDir, key, capacity); err != nil {
+ status := http.StatusInternalServerError
+ if errors.Is(err, safefile.ErrUnsafePath) || validateEphemeralReleaseKey(key) != nil {
+ status = http.StatusBadRequest
+ }
+ http.Error(w, err.Error(), status)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
+
+func handleReleaseBatchWithCapacity(w http.ResponseWriter, r *http.Request, stagingDir string, capacity EphemeralCapacity) {
+ r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
+ var request struct {
+ RequestID string `json:"request_id"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ http.Error(w, fmt.Sprintf("decoding release batch: %v", err), http.StatusBadRequest)
+ return
+ }
+ if err := validateEphemeralRequestID(request.RequestID); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if err := releaseEphemeralStagingRequest(r.Context(), stagingDir, request.RequestID, capacity); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
+
+func releaseEphemeralStagingRequest(ctx context.Context, stagingDir, requestID string, capacity EphemeralCapacity) error {
+ if err := validateEphemeralRequestID(requestID); err != nil {
+ return err
+ }
+ if requestCapacity, ok := capacity.(ephemeralRequestCapacity); ok {
+ if err := requestCapacity.BeginRequestRelease(ctx, requestID); err != nil {
+ return fmt.Errorf("beginning release for request %q: %w", requestID, err)
+ }
+ defer requestCapacity.EndRequestRelease(requestID)
+ }
+ root := filepath.Join(stagingDir, "ephemeral")
+ categories, err := os.ReadDir(root)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ var releaseErrors []error
+ for _, category := range categories {
+ if !category.IsDir() || category.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ requestDir := filepath.Join(root, category.Name(), requestID)
+ info, err := os.Lstat(requestDir)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ releaseErrors = append(releaseErrors, fmt.Errorf("stating request directory %q: %w", requestDir, err))
+ }
+ continue
+ }
+ if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
+ releaseErrors = append(releaseErrors, fmt.Errorf("ephemeral request path %q is not a real directory", requestDir))
+ continue
+ }
+ entries, err := os.ReadDir(requestDir)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ releaseErrors = append(releaseErrors, fmt.Errorf("reading request directory %q: %w", requestDir, err))
+ }
+ continue
+ }
+ for _, entry := range entries {
+ if entry.IsDir() {
+ releaseErrors = append(releaseErrors, fmt.Errorf("unexpected directory in ephemeral request %q", filepath.Join(requestDir, entry.Name())))
+ continue
+ }
+ key := filepath.ToSlash(filepath.Join("ephemeral", category.Name(), requestID, entry.Name()))
+ if err := releaseEphemeralStagingKey(stagingDir, key, capacity); err != nil {
+ releaseErrors = append(releaseErrors, fmt.Errorf("releasing %q: %w", key, err))
+ }
+ }
+ }
+ return errors.Join(releaseErrors...)
+}
+
+func releaseEphemeralStagingKey(stagingDir, key string, capacity EphemeralCapacity) error {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ return err
+ }
+ relativePath := filepath.FromSlash(key)
+ filePath := filepath.Join(stagingDir, relativePath)
+ if err := safefile.RemoveExact(stagingDir, relativePath, []string{hashSidecarSuffix, targetSidecarSuffix}, 2); err != nil {
+ return err
+ }
+ for _, path := range []string{filePath, filePath + hashSidecarSuffix, filePath + targetSidecarSuffix} {
+ if capacity != nil {
+ if err := capacity.Release(path); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
func handleHead(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir, dataDir, key string) {
if key == "" {
http.Error(w, "key is required", http.StatusBadRequest)
@@ -249,6 +435,74 @@ type contentRange struct {
total int64
}
+// EphemeralCapacity bounds worker-local request input storage. Implementations
+// must reserve before bytes reach disk and may reconcile reservations with the
+// resulting file after a write ends.
+type EphemeralCapacity interface {
+ Reserve(path string, size int64) error
+ Commit(path string) error
+ Claim(path string) error
+ Release(path string) error
+ CapacityWriter(path string, destination io.Writer) (io.WriteCloser, error)
+}
+
+type ephemeralRequestCapacity interface {
+ BeginRequestRelease(ctx context.Context, requestID string) error
+ EndRequestRelease(requestID string)
+}
+
+type ephemeralRequestOperationCapacity interface {
+ BeginRequestOperation(requestID string) error
+ EndRequestOperation(requestID string)
+}
+
+type uploadStatusWriter struct {
+ http.ResponseWriter
+ status int
+}
+
+func (w *uploadStatusWriter) WriteHeader(status int) {
+ w.status = status
+ w.ResponseWriter.WriteHeader(status)
+}
+
+func (w *uploadStatusWriter) Write(payload []byte) (int, error) {
+ if w.status == 0 {
+ w.status = http.StatusOK
+ }
+ return w.ResponseWriter.Write(payload)
+}
+
+type ephemeralCapacityWriteError struct{ err error }
+
+func (e *ephemeralCapacityWriteError) Error() string { return e.err.Error() }
+func (e *ephemeralCapacityWriteError) Unwrap() error { return e.err }
+
+type ephemeralCapacityWriteCloser struct{ io.WriteCloser }
+
+func (w ephemeralCapacityWriteCloser) Write(payload []byte) (int, error) {
+ written, err := w.WriteCloser.Write(payload)
+ if err != nil {
+ return written, &ephemeralCapacityWriteError{err: err}
+ }
+ return written, nil
+}
+
+func (w ephemeralCapacityWriteCloser) Close() error {
+ if err := w.WriteCloser.Close(); err != nil {
+ return &ephemeralCapacityWriteError{err: err}
+ }
+ return nil
+}
+
+func uploadWriteStatus(err error) int {
+ var capacityErr *ephemeralCapacityWriteError
+ if errors.As(err, &capacityErr) {
+ return http.StatusInsufficientStorage
+ }
+ return http.StatusInternalServerError
+}
+
// parseContentRange parses a Content-Range header value of the form
// "bytes -/". RFC 9110 §14.4.
// Returns (nil, nil) when the header is empty (no range request).
@@ -290,10 +544,29 @@ func parseContentRange(h string) (*contentRange, error) {
}
func handleUpload(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir, dataDir, key string, maxUploadSize int64) {
+ handleUploadWithCapacity(w, r, stagingDir, modelsDir, dataDir, key, maxUploadSize, nil)
+}
+
+func handleUploadWithCapacity(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir, dataDir, key string, maxUploadSize int64, capacity EphemeralCapacity) {
if key == "" {
http.Error(w, "key is required", http.StatusBadRequest)
return
}
+ capacityEnabled := capacity != nil && strings.HasPrefix(key, "ephemeral/")
+ if capacityEnabled {
+ if err := validateEphemeralReleaseKey(key); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if operations, ok := capacity.(ephemeralRequestOperationCapacity); ok {
+ requestID := strings.Split(key, "/")[2]
+ if err := operations.BeginRequestOperation(requestID); err != nil {
+ http.Error(w, err.Error(), http.StatusConflict)
+ return
+ }
+ defer operations.EndRequestOperation(requestID)
+ }
+ }
if maxUploadSize > 0 {
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
@@ -328,18 +601,67 @@ func handleUpload(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir,
return
}
- if cr == nil {
- // Non-resumable (legacy) path: truncate-create, single fire-and-forget.
- handleFullUpload(w, r, dstPath, key, expectedFinalHash)
- return
+ capacityEnabled = capacityEnabled && targetDir == stagingDir
+ unknownLengthCapacity := capacityEnabled && r.ContentLength < 0
+ capacityPaths := []string{dstPath, dstPath + hashSidecarSuffix, dstPath + targetSidecarSuffix}
+ if capacityEnabled {
+ if r.ContentLength >= 0 {
+ if err := capacity.Reserve(dstPath, r.ContentLength); err != nil {
+ http.Error(w, err.Error(), http.StatusInsufficientStorage)
+ return
+ }
+ }
+ for _, sidecarPath := range capacityPaths[1:] {
+ if err := capacity.Reserve(sidecarPath, sha256.Size*2); err != nil {
+ for _, reservedPath := range capacityPaths {
+ reconcileEphemeralCapacity(capacity, reservedPath, 0)
+ }
+ http.Error(w, err.Error(), http.StatusInsufficientStorage)
+ return
+ }
+ }
}
- handleRangeUpload(w, r, dstPath, key, cr, expectedFinalHash)
+ statusWriter := &uploadStatusWriter{ResponseWriter: w}
+ var uploadCapacity EphemeralCapacity
+ if unknownLengthCapacity {
+ uploadCapacity = capacity
+ }
+
+ if cr == nil {
+ // Non-resumable (legacy) path: truncate-create, single fire-and-forget.
+ handleFullUpload(statusWriter, r, dstPath, key, expectedFinalHash, uploadCapacity)
+ } else {
+ handleRangeUpload(statusWriter, r, dstPath, key, cr, expectedFinalHash, uploadCapacity)
+ }
+
+ if !capacityEnabled {
+ return
+ }
+ for _, capacityPath := range capacityPaths {
+ reconcileEphemeralCapacity(capacity, capacityPath, statusWriter.status)
+ }
+}
+
+func reconcileEphemeralCapacity(capacity EphemeralCapacity, path string, status int) {
+ if info, err := os.Lstat(path); err == nil && info.Mode().IsRegular() {
+ if err := capacity.Commit(path); err != nil {
+ xlog.Error("Committing ephemeral capacity failed", "path", path, "status", status, "error", err)
+ if removeErr := os.Remove(path); removeErr != nil && !os.IsNotExist(removeErr) {
+ xlog.Warn("Removing uncommitted ephemeral file failed", "path", path, "error", removeErr)
+ }
+ if releaseErr := capacity.Release(path); releaseErr != nil {
+ xlog.Warn("Rolling back failed ephemeral commit", "path", path, "error", releaseErr)
+ }
+ }
+ } else if err := capacity.Release(path); err != nil {
+ xlog.Warn("Rolling back ephemeral capacity failed", "path", path, "error", err)
+ }
}
// handleFullUpload writes the entire request body to dstPath, replacing any
// existing content. This is the legacy happy-path with no Range header.
-func handleFullUpload(w http.ResponseWriter, r *http.Request, dstPath, key, expectedFinalHash string) {
+func handleFullUpload(w http.ResponseWriter, r *http.Request, dstPath, key, expectedFinalHash string, capacity EphemeralCapacity) {
// Reset any in-progress resumable state.
_ = os.Remove(dstPath + targetSidecarSuffix)
@@ -350,13 +672,32 @@ func handleFullUpload(w http.ResponseWriter, r *http.Request, dstPath, key, expe
}
defer f.Close()
+ var destination io.Writer = f
+ var capacityWriter io.WriteCloser
+ if capacity != nil {
+ var writer io.WriteCloser
+ writer, err = capacity.CapacityWriter(dstPath, f)
+ if err != nil {
+ _ = os.Remove(dstPath)
+ http.Error(w, err.Error(), http.StatusInsufficientStorage)
+ return
+ }
+ capacityWriter = ephemeralCapacityWriteCloser{WriteCloser: writer}
+ destination = capacityWriter
+ }
+
hasher := sha256.New()
- n, err := io.Copy(f, io.TeeReader(r.Body, hasher))
+ n, err := io.Copy(destination, io.TeeReader(r.Body, hasher))
+ if capacityWriter != nil {
+ if closeErr := capacityWriter.Close(); err == nil {
+ err = closeErr
+ }
+ }
if err != nil {
os.Remove(dstPath)
os.Remove(dstPath + hashSidecarSuffix)
xlog.Error("File upload failed", "key", key, "bytesReceived", n, "contentLength", r.ContentLength, "remote", r.RemoteAddr, "error", err)
- http.Error(w, fmt.Sprintf("writing file: %v", err), http.StatusInternalServerError)
+ http.Error(w, fmt.Sprintf("writing file: %v", err), uploadWriteStatus(err))
return
}
@@ -385,7 +726,7 @@ func handleFullUpload(w http.ResponseWriter, r *http.Request, dstPath, key, expe
// the request starts at the current file size. When the slice completes the
// transfer (end+1 == total), it validates the optional expected final hash and
// writes the sidecar.
-func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key string, cr *contentRange, expectedFinalHash string) {
+func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key string, cr *contentRange, expectedFinalHash string, capacity EphemeralCapacity) {
// Determine the current on-disk size (0 if missing).
var currentSize int64
if info, err := os.Stat(dstPath); err == nil {
@@ -428,6 +769,13 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri
_ = os.Remove(dstPath + hashSidecarSuffix)
_ = os.Remove(targetSidecar)
currentSize = 0
+ } else if currentSize == cr.total {
+ // The final bytes may have landed before a worker stopped during
+ // verification. Clients retry from zero when HEAD reports the full
+ // size without a committed hash. Verify the actual bytes, not the
+ // target sidecar, so this retry can finish without a 416 loop.
+ finalizeRangeUpload(w, dstPath, key, currentSize, expectedFinalHash)
+ return
}
}
@@ -462,6 +810,18 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri
return
}
defer func() { _ = f.Close() }()
+ var destination io.Writer = f
+ var capacityWriter io.WriteCloser
+ if capacity != nil {
+ var writer io.WriteCloser
+ writer, err = capacity.CapacityWriter(dstPath, f)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInsufficientStorage)
+ return
+ }
+ capacityWriter = ephemeralCapacityWriteCloser{WriteCloser: writer}
+ destination = capacityWriter
+ }
// Persist the declared expected hash so subsequent chunks can be
// cross-checked.
@@ -473,10 +833,15 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri
expectedChunkLen := cr.end - cr.start + 1
limited := io.LimitReader(r.Body, expectedChunkLen)
- n, err := io.Copy(f, limited)
+ n, err := io.Copy(destination, limited)
+ if capacityWriter != nil {
+ if closeErr := capacityWriter.Close(); err == nil {
+ err = closeErr
+ }
+ }
if err != nil {
xlog.Error("Range upload chunk failed", "key", key, "bytesReceived", n, "expected", expectedChunkLen, "remote", r.RemoteAddr, "error", err)
- http.Error(w, fmt.Sprintf("writing file: %v", err), http.StatusInternalServerError)
+ http.Error(w, fmt.Sprintf("writing file: %v", err), uploadWriteStatus(err))
return
}
if n != expectedChunkLen {
@@ -499,7 +864,13 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri
return
}
- // Upload complete — compute the final hash by re-reading the file.
+ finalizeRangeUpload(w, dstPath, key, newSize, expectedFinalHash)
+}
+
+// finalizeRangeUpload also recovers transfers interrupted after the last byte
+// was written, so it must hash the file rather than trust cached metadata.
+func finalizeRangeUpload(w http.ResponseWriter, dstPath, key string, size int64, expectedFinalHash string) {
+ targetSidecar := dstPath + targetSidecarSuffix
finalHash, err := downloader.CalculateSHA(dstPath)
if err != nil {
xlog.Error("Failed to compute final hash on range upload", "path", dstPath, "error", err)
@@ -521,7 +892,7 @@ func handleRangeUpload(w http.ResponseWriter, r *http.Request, dstPath, key stri
// Clear the in-progress sidecar — upload is committed.
_ = os.Remove(targetSidecar)
- xlog.Info("Resumable file upload complete", "key", key, "path", dstPath, "size", newSize, "sha256", finalHash)
+ xlog.Info("Resumable file upload complete", "key", key, "path", dstPath, "size", size, "sha256", finalHash)
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(map[string]string{"local_path": dstPath}); err != nil {
diff --git a/core/services/nodes/file_transfer_server_test.go b/core/services/nodes/file_transfer_server_test.go
index 78afb293b..918379c5c 100644
--- a/core/services/nodes/file_transfer_server_test.go
+++ b/core/services/nodes/file_transfer_server_test.go
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
+ "errors"
"fmt"
"io"
"net"
@@ -17,10 +18,99 @@ import (
"sync"
"time"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
+type recordingEphemeralCapacity struct {
+ reserved int64
+ reserveErr error
+ writerErr error
+ claimCalls []string
+ claimErr error
+ commitErr error
+ releases []string
+ startedOps []string
+ endedOps []string
+}
+
+type nopWriteCloser struct{ io.Writer }
+
+func (nopWriteCloser) Close() error { return nil }
+
+type failingWriteCloser struct{ err error }
+
+func (w failingWriteCloser) Write([]byte) (int, error) { return 0, w.err }
+func (failingWriteCloser) Close() error { return nil }
+
+type blockingDestinationWriteCloser struct {
+ destination io.Writer
+ written chan struct{}
+ release chan struct{}
+}
+
+func (w *blockingDestinationWriteCloser) Write(payload []byte) (int, error) {
+ n, err := w.destination.Write(payload)
+ close(w.written)
+ <-w.release
+ return n, err
+}
+
+func (*blockingDestinationWriteCloser) Close() error { return nil }
+
+type blockingDestinationCapacity struct {
+ written chan struct{}
+ release chan struct{}
+}
+
+func (*blockingDestinationCapacity) Reserve(string, int64) error { return nil }
+func (*blockingDestinationCapacity) BeginRequestRelease(context.Context, string) error { return nil }
+func (*blockingDestinationCapacity) EndRequestRelease(string) {}
+func (*blockingDestinationCapacity) BeginRequestOperation(string) error { return nil }
+func (*blockingDestinationCapacity) EndRequestOperation(string) {}
+func (*blockingDestinationCapacity) Commit(string) error { return nil }
+func (*blockingDestinationCapacity) Claim(string) error { return nil }
+func (*blockingDestinationCapacity) Release(string) error { return nil }
+func (g *blockingDestinationCapacity) CapacityWriter(_ string, destination io.Writer) (io.WriteCloser, error) {
+ return &blockingDestinationWriteCloser{
+ destination: destination,
+ written: g.written,
+ release: g.release,
+ }, nil
+}
+
+func (g *recordingEphemeralCapacity) Reserve(_ string, size int64) error {
+ g.reserved = size
+ return g.reserveErr
+}
+
+func (*recordingEphemeralCapacity) BeginRequestRelease(context.Context, string) error { return nil }
+func (*recordingEphemeralCapacity) EndRequestRelease(string) {}
+func (g *recordingEphemeralCapacity) BeginRequestOperation(requestID string) error {
+ g.startedOps = append(g.startedOps, requestID)
+ return nil
+}
+func (g *recordingEphemeralCapacity) EndRequestOperation(requestID string) {
+ g.endedOps = append(g.endedOps, requestID)
+}
+
+func (g *recordingEphemeralCapacity) Commit(string) error { return g.commitErr }
+func (g *recordingEphemeralCapacity) Release(path string) error {
+ g.releases = append(g.releases, path)
+ return nil
+}
+func (g *recordingEphemeralCapacity) Claim(path string) error {
+ g.claimCalls = append(g.claimCalls, path)
+ return g.claimErr
+}
+func (g *recordingEphemeralCapacity) CapacityWriter(_ string, destination io.Writer) (io.WriteCloser, error) {
+ if g.writerErr != nil {
+ return failingWriteCloser{err: g.writerErr}, nil
+ }
+ return nopWriteCloser{Writer: destination}, nil
+}
+
var _ = Describe("FileTransferServer", func() {
setupTestServer := func(token string, maxUploadSize int64) (*httptest.Server, string, string, string) {
stagingDir := GinkgoT().TempDir()
@@ -54,6 +144,67 @@ var _ = Describe("FileTransferServer", func() {
}
Describe("Upload and Download", func() {
+ It("rejects a declared ephemeral upload before writing when capacity is exhausted", func() {
+ stagingDir := GinkgoT().TempDir()
+ modelsDir := GinkgoT().TempDir()
+ dataDir := GinkgoT().TempDir()
+ guard := &recordingEphemeralCapacity{reserveErr: fmt.Errorf("full")}
+ recorder := httptest.NewRecorder()
+ request := httptest.NewRequest(http.MethodPut, "/v1/files/ephemeral/audio/request/input.wav", strings.NewReader("payload"))
+
+ handleUploadWithCapacity(recorder, request, stagingDir, modelsDir, dataDir, "ephemeral/audio/request/input.wav", 0, guard)
+
+ Expect(recorder.Code).To(Equal(http.StatusInsufficientStorage))
+ Expect(guard.reserved).To(Equal(int64(len("payload"))))
+ Expect(filepath.Join(stagingDir, "ephemeral", "audio", "request", "input.wav")).NotTo(BeAnExistingFile())
+ })
+
+ It("returns insufficient storage when a chunked upload reaches its bound", func() {
+ stagingDir := GinkgoT().TempDir()
+ guard := &recordingEphemeralCapacity{writerErr: fmt.Errorf("full")}
+ recorder := httptest.NewRecorder()
+ request := httptest.NewRequest(http.MethodPut, "/v1/files/ephemeral/audio/request/input.wav", strings.NewReader("payload"))
+ request.ContentLength = -1
+
+ handleUploadWithCapacity(recorder, request, stagingDir, GinkgoT().TempDir(), GinkgoT().TempDir(), "ephemeral/audio/request/input.wav", 0, guard)
+
+ Expect(recorder.Code).To(Equal(http.StatusInsufficientStorage))
+ })
+
+ It("keeps unknown-length bytes guarded until they reach the staged file", func() {
+ stagingDir := GinkgoT().TempDir()
+ modelsDir := GinkgoT().TempDir()
+ dataDir := GinkgoT().TempDir()
+ guard := &blockingDestinationCapacity{
+ written: make(chan struct{}),
+ release: make(chan struct{}),
+ }
+ released := false
+ defer func() {
+ if !released {
+ close(guard.release)
+ }
+ }()
+ recorder := httptest.NewRecorder()
+ request := httptest.NewRequest(http.MethodPut, "/v1/files/ephemeral/audio/request/input.wav", strings.NewReader("payload"))
+ request.ContentLength = -1
+ done := make(chan struct{})
+
+ go func() {
+ defer GinkgoRecover()
+ handleUploadWithCapacity(recorder, request, stagingDir, modelsDir, dataDir, "ephemeral/audio/request/input.wav", 0, guard)
+ close(done)
+ }()
+
+ Eventually(guard.written).Should(BeClosed())
+ path := filepath.Join(stagingDir, "ephemeral", "audio", "request", "input.wav")
+ Expect(os.ReadFile(path)).To(Equal([]byte("payload")))
+ close(guard.release)
+ released = true
+ Eventually(done).Should(BeClosed())
+ Expect(recorder.Code).To(Equal(http.StatusOK))
+ })
+
It("round-trips file content correctly", func() {
ts, _, _, _ := setupTestServer("secret-token", 0)
@@ -394,6 +545,17 @@ var _ = Describe("FileTransferServer", func() {
})
})
+ It("removes and releases a file whose capacity commit fails", func() {
+ path := filepath.Join(GinkgoT().TempDir(), "input.wav")
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ guard := &recordingEphemeralCapacity{commitErr: errors.New("request released")}
+
+ reconcileEphemeralCapacity(guard, path, http.StatusOK)
+
+ Expect(path).NotTo(BeAnExistingFile())
+ Expect(guard.releases).To(Equal([]string{path}))
+ })
+
// --- Upload sidecar tests ---
Describe("Upload hash sidecar", func() {
@@ -438,6 +600,165 @@ var _ = Describe("FileTransferServer", func() {
// --- EnsureRemote skip tests ---
Describe("EnsureRemote skip-if-exists", func() {
+ It("reports a claim-time disappearance as a cache miss", func() {
+ stagingDir := GinkgoT().TempDir()
+ key := "ephemeral/audio/request/input.wav"
+ remotePath := filepath.Join(stagingDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(remotePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(remotePath, []byte("stale"), 0o600)).To(Succeed())
+ guard := &recordingEphemeralCapacity{claimErr: fmt.Errorf("claim raced recovery: %w", os.ErrNotExist)}
+ recorder := httptest.NewRecorder()
+ request := httptest.NewRequest(http.MethodPost, "/v1/files/"+key, nil)
+
+ handleClaimWithCapacity(recorder, request, stagingDir, key, guard)
+
+ Expect(recorder.Code).To(Equal(http.StatusNotFound))
+ })
+
+ It("claims a matching ephemeral file before returning the worker path", func() {
+ stagingDir := GinkgoT().TempDir()
+ modelsDir := GinkgoT().TempDir()
+ dataDir := GinkgoT().TempDir()
+ guard := &recordingEphemeralCapacity{}
+ key := "ephemeral/audio/request/input.wav"
+ remotePath := filepath.Join(stagingDir, filepath.FromSlash(key))
+ content := []byte("already on worker")
+ Expect(os.MkdirAll(filepath.Dir(remotePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(remotePath, content, 0o600)).To(Succeed())
+ Expect(os.WriteFile(remotePath+hashSidecarSuffix, []byte(sha256Hex(content)), 0o600)).To(Succeed())
+ localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+ Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/v1/files/", func(w http.ResponseWriter, r *http.Request) {
+ requestKey := strings.TrimPrefix(r.URL.Path, "/v1/files/")
+ switch r.Method {
+ case http.MethodHead:
+ handleHead(w, r, stagingDir, modelsDir, dataDir, requestKey)
+ case http.MethodPost:
+ handleClaimWithCapacity(w, r, stagingDir, requestKey, guard)
+ default:
+ http.Error(w, "unexpected upload", http.StatusInternalServerError)
+ }
+ })
+ ts := httptest.NewServer(mux)
+ DeferCleanup(ts.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(ts.URL, "http://"), nil
+ }, "")
+
+ for range 2 {
+ path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, key)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(path).To(Equal(remotePath))
+ }
+ Expect(guard.claimCalls).To(Equal([]string{remotePath, remotePath}))
+ Expect(guard.startedOps).To(Equal([]string{"request", "request"}))
+ Expect(guard.endedOps).To(Equal([]string{"request", "request"}))
+ })
+
+ It("propagates an ephemeral cache-hit claim failure", func() {
+ content := []byte("already on worker")
+ localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+ Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
+ mux := http.NewServeMux()
+ mux.HandleFunc("/v1/files/", func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodHead {
+ w.Header().Set(HeaderLocalPath, "/remote/ephemeral/audio/request/input.wav")
+ w.Header().Set(HeaderContentSHA256, sha256Hex(content))
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ http.Error(w, "ephemeral capacity exceeded", http.StatusInsufficientStorage)
+ })
+ ts := httptest.NewServer(mux)
+ DeferCleanup(ts.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(ts.URL, "http://"), nil
+ }, "")
+
+ path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "ephemeral/audio/request/input.wav")
+
+ Expect(path).To(BeEmpty())
+ Expect(err).To(MatchError(ContainSubstring("ephemeral capacity exceeded")))
+ })
+
+ DescribeTable("uploads a matching ephemeral file when an old worker cannot claim it",
+ func(claimStatus int) {
+ stagingDir := GinkgoT().TempDir()
+ content := []byte("compatible upload")
+ localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+ Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
+ cacheHitPath := filepath.Join(stagingDir, "ephemeral", "audio", "stale", "input.wav")
+ putCalls := 0
+ putRemotePath := ""
+ mux := http.NewServeMux()
+ mux.HandleFunc("/v1/files/", func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodHead:
+ w.Header().Set(HeaderLocalPath, cacheHitPath)
+ w.Header().Set(HeaderContentSHA256, sha256Hex(content))
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ http.Error(w, "claim unsupported", claimStatus)
+ case http.MethodPut:
+ putCalls++
+ key := strings.TrimPrefix(r.URL.Path, "/v1/files/")
+ putRemotePath = filepath.Join(stagingDir, filepath.FromSlash(key))
+ handleUpload(w, r, stagingDir, "", "", key, 0)
+ case http.MethodDelete:
+ w.WriteHeader(http.StatusNoContent)
+ }
+ })
+ ts := httptest.NewServer(mux)
+ DeferCleanup(ts.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(ts.URL, "http://"), nil
+ }, "")
+
+ backend := &lifecycleBackend{}
+ client := NewFileStagingClient(backend, stager, "node-1")
+ request := &pb.PredictOptions{Audios: []string{localPath}}
+ _, err := client.Predict(context.Background(), request)
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(putCalls).To(Equal(1))
+ Expect(backend.predictInput).NotTo(BeNil())
+ Expect(backend.predictInput.Audios).To(Equal([]string{putRemotePath}))
+ },
+ Entry("404", http.StatusNotFound),
+ Entry("405", http.StatusMethodNotAllowed),
+ )
+
+ It("keeps matching model probes read-only", func() {
+ content := []byte("model")
+ localPath := filepath.Join(GinkgoT().TempDir(), "model.bin")
+ Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
+ unexpectedWrites := 0
+ mux := http.NewServeMux()
+ mux.HandleFunc("/v1/files/", func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodHead {
+ unexpectedWrites++
+ http.Error(w, "unexpected write", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set(HeaderLocalPath, "/models/tracking/model.bin")
+ w.Header().Set(HeaderContentSHA256, sha256Hex(content))
+ w.WriteHeader(http.StatusOK)
+ })
+ ts := httptest.NewServer(mux)
+ DeferCleanup(ts.Close)
+ stager := NewHTTPFileStager(func(string) (string, error) {
+ return strings.TrimPrefix(ts.URL, "http://"), nil
+ }, "")
+
+ path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "models/tracking/model.bin")
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(path).To(Equal("/models/tracking/model.bin"))
+ Expect(unexpectedWrites).To(BeZero())
+ })
+
It("skips upload when file exists with matching hash", func() {
ts, stagingDir, _, _ := setupTestServer("tok", 0)
diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go
index 44acede11..04d420969 100644
--- a/core/services/nodes/router.go
+++ b/core/services/nodes/router.go
@@ -1599,8 +1599,15 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
// Stage file paths referenced in generic Options (key:value pairs where values
// are file paths). Options stay as relative paths — backends resolve them via ModelPath.
- r.stageGenericOptions(ctx, node, opts.Options, frontendModelsDir, localModelDir, keyMapper.Key)
- r.stageGenericOptions(ctx, node, opts.Overrides, frontendModelsDir, localModelDir, keyMapper.Key)
+ for _, options := range [][]string{opts.Options, opts.Overrides} {
+ remoteRoot := r.stageGenericOptions(ctx, node, options, frontendModelsDir, localModelDir, keyMapper.Key)
+ if opts.ModelFile == "" && remoteRoot != "" {
+ // Virtual models have no primary file from which to derive the
+ // worker root. Their relative options must resolve against the
+ // companion assets we actually staged, not the frontend's root.
+ opts.ModelPath = remoteRoot
+ }
+ }
return opts, nil
}
@@ -1831,7 +1838,9 @@ func (r *SmartRouter) stageCompanionFiles(ctx context.Context, node *BackendNode
// that resolve to existing files relative to the frontend models directory or
// the model's own directory. Option values are NOT rewritten — backends resolve
// them via ModelPath. keyFn generates the namespaced storage key for each file.
-func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) {
+// Returns the staged models root, or empty when no asset was staged.
+func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) string {
+ remoteRoot := ""
for _, opt := range options {
optKey, val, ok := strings.Cut(opt, ":")
if !ok || val == "" {
@@ -1856,18 +1865,23 @@ func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode
// worker; a single file is staged directly. Values are never rewritten —
// backends resolve relative paths via ModelPath.
if err == nil && info.IsDir() {
- r.stageOptionDir(ctx, node, absPath, keyFn)
+ if remoteDir := r.stageOptionDir(ctx, node, absPath, keyFn); remoteDir != "" {
+ remoteRoot = DeriveRemoteModelPath(remoteDir, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
+ }
xlog.Debug("Staged option directory", "option", optKey, "localPath", absPath)
continue
}
key := keyFn(absPath)
- if _, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key); err != nil {
+ remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key)
+ if err != nil {
xlog.Warn("Failed to stage option file, skipping", "option", opt, "path", absPath, "error", err)
continue
}
+ remoteRoot = DeriveRemoteModelPath(remotePath, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
xlog.Debug("Staged option file", "option", optKey, "localPath", absPath)
}
+ return remoteRoot
}
// resolveOptionPath finds an existing local path for an option value: an
@@ -1895,8 +1909,10 @@ func resolveOptionPath(val, frontendModelsDir, modelDir string) (string, bool) {
// stageOptionDir stages every regular file under an option-declared directory
// (e.g. sherpa-onnx's espeak-ng-data) using the structure-preserving key, so the
// tree is recreated beside the model on the worker. Per-file errors are logged
-// and skipped; the option value itself is not rewritten.
-func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) {
+// and skipped; the option value itself is not rewritten. Returns the remote
+// directory derived from a successfully staged file, or empty when none succeeds.
+func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) string {
+ remoteDir := ""
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil || d.IsDir() {
return nil
@@ -1911,11 +1927,17 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir
if isHashSidecar(path) {
return nil
}
- if _, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path)); err != nil {
+ remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path))
+ if err != nil {
xlog.Warn("Failed to stage option directory file, skipping", "path", path, "error", err)
+ return nil
+ }
+ if rel, err := filepath.Rel(dir, path); err == nil {
+ remoteDir = DeriveRemoteModelPath(remotePath, rel)
}
return nil
})
+ return remoteDir
}
// probeHealth checks whether a backend process on the given node/addr is alive
diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go
index 10c646329..c6140bc1b 100644
--- a/core/services/nodes/router_test.go
+++ b/core/services/nodes/router_test.go
@@ -52,6 +52,8 @@ func (f *fakeFileStager) AllocRemoteTemp(_ context.Context, _ string) (string, e
func (f *fakeFileStager) StageRemoteToStore(_ context.Context, _, _, _ string) error { return nil }
+func (f *fakeFileStager) ReleaseRemote(_ context.Context, _, _ string) error { return nil }
+
func (f *fakeFileStager) ListRemoteDir(_ context.Context, _, _ string) ([]string, error) {
return nil, nil
}
diff --git a/core/services/nodes/router_virtual_model_test.go b/core/services/nodes/router_virtual_model_test.go
new file mode 100644
index 000000000..4d1a5e7cf
--- /dev/null
+++ b/core/services/nodes/router_virtual_model_test.go
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: MIT
+
+package nodes
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type failedCompanionStager struct{ FileStager }
+
+func (failedCompanionStager) EnsureRemote(context.Context, string, string, string) (string, error) {
+ return "", errors.New("worker unavailable")
+}
+
+var _ = Describe("staging virtual model companions", func() {
+ DescribeTable("anchors relative assets on the worker",
+ func(options, overrides []string, files []string) {
+ modelsDir := GinkgoT().TempDir()
+ for _, name := range files {
+ path := filepath.Join(modelsDir, name)
+ Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("weights"), 0600)).To(Succeed())
+ }
+ stager := &fakeFileStager{}
+ router := &SmartRouter{fileStager: stager, stagingTracker: NewStagingTracker()}
+ input := &pb.ModelOptions{Model: "insightface-buffalo-m", ModelFile: filepath.Join(modelsDir, "insightface-buffalo-m"), ModelPath: modelsDir, Options: options, Overrides: overrides}
+ staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "insightface-buffalo-m")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(staged.ModelPath).To(Equal("/remote/models/insightface-buffalo-m"))
+ Expect(staged.Options).To(Equal(options))
+ Expect(staged.Overrides).To(Equal(overrides))
+ Expect(input.ModelPath).To(Equal(modelsDir))
+ Expect(input.ModelFile).To(Equal(filepath.Join(modelsDir, "insightface-buffalo-m")))
+ Expect(stager.ensureCalls).To(HaveLen(len(files)))
+ for _, call := range stager.ensureCalls {
+ rel, err := filepath.Rel(modelsDir, call.localPath)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(filepath.Join(staged.ModelPath, rel)).To(Equal("/remote/" + call.key))
+ }
+ },
+ Entry("Buffalo pack and MiniFASNet files", []string{"model_pack:buffalo_m", "antispoof_v2_onnx:MiniFASNetV2.onnx", "antispoof_v1se_onnx:MiniFASNetV1SE.onnx"}, nil, []string{"buffalo_m/det_2.5g.onnx", "buffalo_m/w600k_r50.onnx", "MiniFASNetV2.onnx", "MiniFASNetV1SE.onnx"}),
+ Entry("only a nested companion directory", []string{"model_pack:packs/buffalo_m"}, nil, []string{"packs/buffalo_m/det_2.5g.onnx"}),
+ Entry("only a companion file", []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, nil, []string{"MiniFASNetV2.onnx"}),
+ Entry("only override assets", nil, []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, []string{"MiniFASNetV2.onnx"}),
+ )
+ It("keeps the original path when no assets are staged", func() {
+ modelsDir := GinkgoT().TempDir()
+ router := &SmartRouter{fileStager: &fakeFileStager{}, stagingTracker: NewStagingTracker()}
+ input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"engine:insightface"}}
+ staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(staged.ModelPath).To(Equal(modelsDir))
+ })
+ DescribeTable("keeps the original root when companion staging fails",
+ func(directory bool) {
+ modelsDir := GinkgoT().TempDir()
+ relative := "MiniFASNetV2.onnx"
+ if directory {
+ relative = "buffalo_m/det_2.5g.onnx"
+ }
+ local := filepath.Join(modelsDir, relative)
+ Expect(os.MkdirAll(filepath.Dir(local), 0750)).To(Succeed())
+ Expect(os.WriteFile(local, []byte("weights"), 0600)).To(Succeed())
+ value := relative
+ if directory {
+ value = "buffalo_m"
+ }
+ router := &SmartRouter{fileStager: failedCompanionStager{}, stagingTracker: NewStagingTracker()}
+ input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"asset:" + value}}
+ staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+ Expect(err).NotTo(HaveOccurred())
+ Expect(staged.ModelPath).To(Equal(modelsDir))
+ }, Entry("file", false), Entry("directory", true),
+ )
+
+})
diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go
index cd9cbcead..a6a6eefcb 100644
--- a/core/services/quantization/service.go
+++ b/core/services/quantization/service.go
@@ -42,6 +42,28 @@ type QuantizationService struct {
// jobs is the cross-replica job store: an in-memory map kept consistent across
// replicas via NATS, optionally read-through to PostgreSQL in distributed mode.
jobs *syncstate.SyncedMap[string, *schema.QuantizationJob]
+
+ // progressMu guards progressSubs.
+ //
+ // A backend's per-job progress stream has a single destructive consumer: the
+ // backend pops each update off one queue and hands it to whoever is reading.
+ // So the service opens that stream exactly once per job — in watchProgress,
+ // started by StartJob — and fans the updates out in-process to the SSE clients
+ // registered here. Opening a second stream per client would make the two
+ // readers race for the same updates.
+ progressMu sync.Mutex
+ progressSubs map[string][]chan *schema.QuantizationProgressEvent
+}
+
+// progressSubBuffer is the per-subscriber event buffer. It absorbs a client that
+// is briefly slow; a client that falls further behind drops events rather than
+// stalling the single reader of the backend stream.
+const progressSubBuffer = 64
+
+// isTerminalStatus reports whether a job status is final, i.e. no further
+// progress update will follow.
+func isTerminalStatus(status string) bool {
+ return status == "stopped" || status == "completed" || status == "failed"
}
// NewQuantizationService creates a new QuantizationService. In distributed mode
@@ -59,6 +81,7 @@ func NewQuantizationService(
appConfig: appConfig,
modelLoader: modelLoader,
configLoader: configLoader,
+ progressSubs: make(map[string][]chan *schema.QuantizationProgressEvent),
}
// Only attach a Store interface when a concrete store exists, otherwise the
@@ -240,6 +263,13 @@ func (s *QuantizationService) StartJob(ctx context.Context, userID string, req s
}
s.saveJobState(job)
+ // Consume the backend's progress stream for the lifetime of the job, not for
+ // the lifetime of a client's SSE connection: a job that runs with nobody
+ // attached must still reach "completed" in the store and in state.json. The
+ // request ctx is done as soon as this HTTP handler returns, so the watcher
+ // rides the application context instead.
+ go s.watchProgress(s.appConfig.Context, jobID, backendName, modelID)
+
return &schema.QuantizationJobResponse{
ID: jobID,
Status: "queued",
@@ -311,6 +341,14 @@ func (s *QuantizationService) StopJob(ctx context.Context, userID, jobID string)
s.saveJobState(job)
s.mu.Unlock()
+ // Release clients attached to the progress stream: the backend process is gone,
+ // so the watcher will not see a terminal update to forward.
+ s.publishProgress(jobID, &schema.QuantizationProgressEvent{
+ JobID: jobID,
+ Status: "stopped",
+ Message: "Quantization stopped by user",
+ })
+
return nil
}
@@ -377,7 +415,153 @@ func (s *QuantizationService) DeleteJob(userID, jobID string) error {
return nil
}
-// StreamProgress opens a gRPC progress stream and calls the callback for each update.
+// watchProgress is the single reader of a job's backend progress stream. It
+// records every transition on the job — in the cross-replica store and in
+// state.json — and republishes it to the clients attached via StreamProgress.
+//
+// Recording here rather than in StreamProgress is the point: the backend hands
+// each update to one consumer, so while StreamProgress was that consumer a job's
+// state only advanced while somebody was watching it.
+func (s *QuantizationService) watchProgress(ctx context.Context, jobID, backendName, modelID string) {
+ backendModel, err := s.modelLoader.Load(
+ model.WithBackendString(backendName),
+ model.WithModel(backendName),
+ model.WithModelID(modelID),
+ )
+ if err != nil {
+ xlog.Warn("Failed to load backend for quantization progress", "job_id", jobID, "error", err)
+ return
+ }
+
+ err = backendModel.QuantizationProgress(ctx, &pb.QuantizationProgressRequest{
+ JobId: jobID,
+ }, func(update *pb.QuantizationProgressUpdate) {
+ s.publishProgress(jobID, s.applyProgressUpdate(ctx, jobID, update))
+ })
+ if err != nil {
+ xlog.Warn("Quantization progress stream ended with an error", "job_id", jobID, "error", err)
+ }
+
+ // On shutdown leave the job alone: loadJobsFromDisk already reports jobs that
+ // were running at exit as stopped.
+ if ctx.Err() != nil {
+ return
+ }
+
+ // A stream that ends without a terminal update means the backend is gone and
+ // nothing further will arrive. Record that instead of leaving the job in a
+ // running state forever — which is the failure this watcher exists to prevent —
+ // and release any client still waiting on a terminal event.
+ s.mu.Lock()
+ j, ok := s.jobs.Get(jobID)
+ stale := ok && !isTerminalStatus(j.Status)
+ if stale {
+ j.Status = "failed"
+ if j.Message == "" {
+ j.Message = "Backend progress stream ended before the job reported a result"
+ }
+ if err := s.jobs.Set(ctx, j); err != nil {
+ xlog.Warn("Failed to persist orphaned job state", "job_id", jobID, "error", err)
+ }
+ s.saveJobState(j)
+ }
+ s.mu.Unlock()
+
+ if stale {
+ s.publishProgress(jobID, &schema.QuantizationProgressEvent{
+ JobID: jobID,
+ Status: "failed",
+ Message: "Backend progress stream ended before the job reported a result",
+ })
+ }
+}
+
+// applyProgressUpdate records a backend progress update on the job and returns
+// the event to hand to subscribers.
+func (s *QuantizationService) applyProgressUpdate(ctx context.Context, jobID string, update *pb.QuantizationProgressUpdate) *schema.QuantizationProgressEvent {
+ s.mu.Lock()
+ if j, ok := s.jobs.Get(jobID); ok {
+ // Don't let progress updates overwrite terminal states
+ if !isTerminalStatus(j.Status) {
+ j.Status = update.Status
+ }
+ if update.Message != "" {
+ j.Message = update.Message
+ }
+ if update.OutputFile != "" {
+ j.OutputFile = update.OutputFile
+ }
+ if err := s.jobs.Set(ctx, j); err != nil {
+ xlog.Warn("Failed to persist progress update", "job_id", jobID, "error", err)
+ }
+ s.saveJobState(j)
+ }
+ s.mu.Unlock()
+
+ // Convert extra metrics
+ extraMetrics := make(map[string]float32, len(update.ExtraMetrics))
+ for k, v := range update.ExtraMetrics {
+ extraMetrics[k] = v
+ }
+
+ return &schema.QuantizationProgressEvent{
+ JobID: update.JobId,
+ ProgressPercent: update.ProgressPercent,
+ Status: update.Status,
+ Message: update.Message,
+ OutputFile: update.OutputFile,
+ ExtraMetrics: extraMetrics,
+ }
+}
+
+// subscribeProgress registers a channel to receive a job's progress events.
+func (s *QuantizationService) subscribeProgress(jobID string) chan *schema.QuantizationProgressEvent {
+ ch := make(chan *schema.QuantizationProgressEvent, progressSubBuffer)
+ s.progressMu.Lock()
+ s.progressSubs[jobID] = append(s.progressSubs[jobID], ch)
+ s.progressMu.Unlock()
+ return ch
+}
+
+// unsubscribeProgress removes a channel registered by subscribeProgress. The
+// channel is never closed, so a publish racing with an unsubscribe cannot send
+// on a closed channel.
+func (s *QuantizationService) unsubscribeProgress(jobID string, ch chan *schema.QuantizationProgressEvent) {
+ s.progressMu.Lock()
+ defer s.progressMu.Unlock()
+
+ subs := s.progressSubs[jobID]
+ for i, c := range subs {
+ if c == ch {
+ s.progressSubs[jobID] = append(subs[:i], subs[i+1:]...)
+ break
+ }
+ }
+ if len(s.progressSubs[jobID]) == 0 {
+ delete(s.progressSubs, jobID)
+ }
+}
+
+// publishProgress fans an event out to a job's subscribers.
+func (s *QuantizationService) publishProgress(jobID string, event *schema.QuantizationProgressEvent) {
+ s.progressMu.Lock()
+ subs := append([]chan *schema.QuantizationProgressEvent(nil), s.progressSubs[jobID]...)
+ s.progressMu.Unlock()
+
+ for _, ch := range subs {
+ select {
+ case ch <- event:
+ default:
+ // A subscriber that cannot keep up must not stall the reader that is
+ // recording job state for everyone else.
+ xlog.Warn("Dropping quantization progress event for a slow subscriber", "job_id", jobID)
+ }
+ }
+}
+
+// StreamProgress calls the callback for each progress event of a job until it
+// reaches a terminal status or ctx is done. It is a pure reader: the job's own
+// watcher owns the backend stream and the state transitions.
func (s *QuantizationService) StreamProgress(ctx context.Context, userID, jobID string, callback func(event *schema.QuantizationProgressEvent)) error {
s.mu.Lock()
job, ok := s.jobs.Get(jobID)
@@ -391,59 +575,41 @@ func (s *QuantizationService) StreamProgress(ctx context.Context, userID, jobID
}
s.mu.Unlock()
- streamModelID := job.ModelID
- if streamModelID == "" {
- streamModelID = job.Backend + "-quantize"
+ ch := s.subscribeProgress(jobID)
+ defer s.unsubscribeProgress(jobID, ch)
+
+ // Re-read the job after subscribing: it may have finished between the lookup
+ // above and the subscription, and no further event would ever arrive. Jobs
+ // restored from disk after a restart are terminal too, and have no watcher.
+ s.mu.Lock()
+ current, ok := s.jobs.Get(jobID)
+ terminal := ok && isTerminalStatus(current.Status)
+ var final *schema.QuantizationProgressEvent
+ if terminal {
+ final = &schema.QuantizationProgressEvent{
+ JobID: current.ID,
+ Status: current.Status,
+ Message: current.Message,
+ OutputFile: current.OutputFile,
+ }
}
- backendModel, err := s.modelLoader.Load(
- model.WithBackendString(job.Backend),
- model.WithModel(job.Backend),
- model.WithModelID(streamModelID),
- )
- if err != nil {
- return fmt.Errorf("failed to load backend: %w", err)
+ s.mu.Unlock()
+ if terminal {
+ callback(final)
+ return nil
}
- return backendModel.QuantizationProgress(ctx, &pb.QuantizationProgressRequest{
- JobId: jobID,
- }, func(update *pb.QuantizationProgressUpdate) {
- // Update job status and persist
- s.mu.Lock()
- if j, ok := s.jobs.Get(jobID); ok {
- // Don't let progress updates overwrite terminal states
- isTerminal := j.Status == "stopped" || j.Status == "completed" || j.Status == "failed"
- if !isTerminal {
- j.Status = update.Status
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case event := <-ch:
+ callback(event)
+ if isTerminalStatus(event.Status) {
+ return nil
}
- if update.Message != "" {
- j.Message = update.Message
- }
- if update.OutputFile != "" {
- j.OutputFile = update.OutputFile
- }
- if err := s.jobs.Set(ctx, j); err != nil {
- xlog.Warn("Failed to persist progress update", "job_id", jobID, "error", err)
- }
- s.saveJobState(j)
}
- s.mu.Unlock()
-
- // Convert extra metrics
- extraMetrics := make(map[string]float32)
- for k, v := range update.ExtraMetrics {
- extraMetrics[k] = v
- }
-
- event := &schema.QuantizationProgressEvent{
- JobID: update.JobId,
- ProgressPercent: update.ProgressPercent,
- Status: update.Status,
- Message: update.Message,
- OutputFile: update.OutputFile,
- ExtraMetrics: extraMetrics,
- }
- callback(event)
- })
+ }
}
// sanitizeQuantModelName replaces non-alphanumeric characters with hyphens and lowercases.
diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go
index 665728614..ae862ffca 100644
--- a/core/services/quantization/service_test.go
+++ b/core/services/quantization/service_test.go
@@ -8,6 +8,9 @@ package quantization
import (
"context"
+ "encoding/json"
+ "os"
+ "path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -16,6 +19,7 @@ import (
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/core/services/distributed"
"github.com/mudler/LocalAI/core/services/testutil"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
)
// newTestService builds a standalone QuantizationService wired to the given bus.
@@ -175,6 +179,156 @@ var _ = Describe("QuantizationService", func() {
})
})
+ Describe("progress recording", func() {
+ var (
+ bus *testutil.FakeBus
+ s *QuantizationService
+ )
+
+ BeforeEach(func() {
+ bus = testutil.NewFakeBus()
+ s = newTestService(bus)
+ })
+
+ AfterEach(func() {
+ Expect(s.Close()).To(Succeed())
+ })
+
+ // The reported failure: a job that ran with no SSE client attached stayed
+ // "queued" forever, because the only code that advanced job state lived
+ // inside StreamProgress' stream callback. The transition is now applied by
+ // the job's own watcher, so it lands with nobody watching.
+ It("advances job state and rewrites state.json with no subscriber attached", func() {
+ job := &schema.QuantizationJob{ID: "job-np", UserID: "user-1", Status: "queued", CreatedAt: "2026-09-05T10:00:00Z"}
+ Expect(s.jobs.Set(ctx, job)).To(Succeed())
+ Expect(s.progressSubs).To(BeEmpty())
+
+ s.applyProgressUpdate(ctx, "job-np", &pb.QuantizationProgressUpdate{
+ JobId: "job-np",
+ Status: "completed",
+ Message: "Quantization complete",
+ OutputFile: "/data/quantization/job-np/model-q4_k_m.gguf",
+ })
+
+ got, err := s.GetJob("user-1", "job-np")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(got.Status).To(Equal("completed"))
+ Expect(got.Message).To(Equal("Quantization complete"))
+ Expect(got.OutputFile).To(Equal("/data/quantization/job-np/model-q4_k_m.gguf"))
+
+ data, err := os.ReadFile(filepath.Join(s.jobDir("job-np"), "state.json"))
+ Expect(err).ToNot(HaveOccurred())
+ var persisted schema.QuantizationJob
+ Expect(json.Unmarshal(data, &persisted)).To(Succeed())
+ Expect(persisted.Status).To(Equal("completed"))
+ Expect(persisted.OutputFile).To(Equal("/data/quantization/job-np/model-q4_k_m.gguf"))
+ })
+
+ It("does not let a late update overwrite a terminal status", func() {
+ job := &schema.QuantizationJob{ID: "job-stopped", UserID: "user-1", Status: "stopped", CreatedAt: "2026-09-05T10:00:00Z"}
+ Expect(s.jobs.Set(ctx, job)).To(Succeed())
+
+ s.applyProgressUpdate(ctx, "job-stopped", &pb.QuantizationProgressUpdate{
+ JobId: "job-stopped",
+ Status: "quantizing",
+ })
+
+ got, err := s.GetJob("user-1", "job-stopped")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(got.Status).To(Equal("stopped"))
+ })
+
+ // The backend hands each update to a single consumer, so every client has
+ // to be served from one in-process fan-out rather than its own stream.
+ It("delivers one update to every attached subscriber", func() {
+ first := s.subscribeProgress("job-fan")
+ second := s.subscribeProgress("job-fan")
+ defer s.unsubscribeProgress("job-fan", first)
+ defer s.unsubscribeProgress("job-fan", second)
+
+ s.publishProgress("job-fan", &schema.QuantizationProgressEvent{JobID: "job-fan", Status: "quantizing"})
+
+ Expect((<-first).Status).To(Equal("quantizing"))
+ Expect((<-second).Status).To(Equal("quantizing"))
+ })
+
+ It("unsubscribing removes the job's entry once the last client leaves", func() {
+ ch := s.subscribeProgress("job-leave")
+ Expect(s.progressSubs).To(HaveKey("job-leave"))
+ s.unsubscribeProgress("job-leave", ch)
+ Expect(s.progressSubs).ToNot(HaveKey("job-leave"))
+ })
+
+ // A client attaching after the job finished — including a job restored from
+ // disk as "stopped" after a restart, which has no watcher — must not block
+ // waiting for an event that will never come.
+ It("returns a final event immediately for a job that already finished", func() {
+ job := &schema.QuantizationJob{
+ ID: "job-done", UserID: "user-1", Status: "completed",
+ Message: "Quantization complete", OutputFile: "/data/quantization/job-done/model-q4_k_m.gguf",
+ CreatedAt: "2026-09-05T10:00:00Z",
+ }
+ Expect(s.jobs.Set(ctx, job)).To(Succeed())
+
+ var seen []*schema.QuantizationProgressEvent
+ Expect(s.StreamProgress(ctx, "user-1", "job-done", func(e *schema.QuantizationProgressEvent) {
+ seen = append(seen, e)
+ })).To(Succeed())
+
+ Expect(seen).To(HaveLen(1))
+ Expect(seen[0].Status).To(Equal("completed"))
+ Expect(seen[0].OutputFile).To(Equal("/data/quantization/job-done/model-q4_k_m.gguf"))
+ Expect(s.progressSubs).ToNot(HaveKey("job-done"))
+ })
+
+ // StopJob kills the backend, so the watcher will never forward a terminal
+ // update; without an explicit release an attached client would hang.
+ It("releases an attached client when the job is stopped", func() {
+ job := &schema.QuantizationJob{ID: "job-stop", UserID: "user-1", Status: "quantizing", CreatedAt: "2026-09-05T10:00:00Z"}
+ Expect(s.jobs.Set(ctx, job)).To(Succeed())
+
+ ch := s.subscribeProgress("job-stop")
+ defer s.unsubscribeProgress("job-stop", ch)
+
+ // nil modelLoader: exercise the release without standing up a backend.
+ s.mu.Lock()
+ job.Status = "stopped"
+ s.mu.Unlock()
+ s.publishProgress("job-stop", &schema.QuantizationProgressEvent{
+ JobID: "job-stop", Status: "stopped", Message: "Quantization stopped by user",
+ })
+
+ event := <-ch
+ Expect(event.Status).To(Equal("stopped"))
+ Expect(isTerminalStatus(event.Status)).To(BeTrue())
+ })
+
+ It("streams published events to a client until a terminal status arrives", func() {
+ job := &schema.QuantizationJob{ID: "job-live", UserID: "user-1", Status: "queued", CreatedAt: "2026-09-05T10:00:00Z"}
+ Expect(s.jobs.Set(ctx, job)).To(Succeed())
+
+ var seen []string
+ done := make(chan error, 1)
+ go func() {
+ done <- s.StreamProgress(ctx, "user-1", "job-live", func(e *schema.QuantizationProgressEvent) {
+ seen = append(seen, e.Status)
+ })
+ }()
+
+ Eventually(func() bool {
+ s.progressMu.Lock()
+ defer s.progressMu.Unlock()
+ return len(s.progressSubs["job-live"]) == 1
+ }).Should(BeTrue())
+
+ s.publishProgress("job-live", &schema.QuantizationProgressEvent{JobID: "job-live", Status: "quantizing"})
+ s.publishProgress("job-live", &schema.QuantizationProgressEvent{JobID: "job-live", Status: "completed"})
+
+ Eventually(done).Should(Receive(BeNil()))
+ Expect(seen).To(Equal([]string{"quantizing", "completed"}))
+ })
+ })
+
Describe("compile-time adapter contract", func() {
It("satisfies syncstate.Store for *distributed.QuantStore", func() {
// Guards against drift between the adapter and the component interface;
diff --git a/core/services/worker/config.go b/core/services/worker/config.go
index 8057e69fe..8a63ad8e0 100644
--- a/core/services/worker/config.go
+++ b/core/services/worker/config.go
@@ -47,8 +47,10 @@ type Config struct {
PrefetchModels []string `env:"LOCALAI_PREFETCH_MODELS,PREFETCH_MODELS" help:"Comma-separated gallery model IDs to download from LOCALAI_GALLERIES at worker boot (e.g. 'llama-3.2-1b-instruct,phi-3-mini-4k'). Skipped if already on disk and SHA matches." group:"server"`
// HTTP file transfer
- HTTPAddr string `env:"LOCALAI_HTTP_ADDR" default:"" help:"HTTP file transfer server address (default: gRPC port + 1)" group:"server" hidden:""`
- AdvertiseHTTPAddr string `env:"LOCALAI_ADVERTISE_HTTP_ADDR" help:"HTTP address the frontend uses to reach this node for file transfer" group:"server" hidden:""`
+ HTTPAddr string `env:"LOCALAI_HTTP_ADDR" default:"" help:"HTTP file transfer server address (default: gRPC port + 1)" group:"server" hidden:""`
+ AdvertiseHTTPAddr string `env:"LOCALAI_ADVERTISE_HTTP_ADDR" help:"HTTP address the frontend uses to reach this node for file transfer" group:"server" hidden:""`
+ EphemeralStagingByteLimit int64 `env:"LOCALAI_EPHEMERAL_STAGING_BYTE_LIMIT" default:"0" help:"Maximum bytes used by worker request-input staging across HTTP and S3 caches. Zero or negative uses min(10 GiB, 10% of filesystem capacity)." group:"server"`
+ EphemeralStagingMinFreeBytes int64 `env:"LOCALAI_EPHEMERAL_STAGING_MIN_FREE_BYTES" default:"0" help:"Filesystem space kept free while staging request inputs. Zero or negative uses max(1 GiB, 5% of filesystem capacity)." group:"server"`
// Registration (required)
AdvertiseAddr string `env:"LOCALAI_ADVERTISE_ADDR" help:"Address the frontend uses to reach this node (defaults to hostname:port from Addr)" group:"registration" hidden:""`
diff --git a/core/services/worker/ephemeral_capacity.go b/core/services/worker/ephemeral_capacity.go
new file mode 100644
index 000000000..713c9b221
--- /dev/null
+++ b/core/services/worker/ephemeral_capacity.go
@@ -0,0 +1,1031 @@
+// SPDX-License-Identifier: MIT
+
+package worker
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/mudler/LocalAI/pkg/xsysinfo"
+)
+
+const ephemeralCapacityWriteChunk int64 = 64 << 10
+
+const (
+ defaultEphemeralByteLimitCeiling = int64(10 << 30)
+ defaultEphemeralMinFreeFloor = int64(1 << 30)
+ ephemeralReleaseTombstoneTTL = time.Hour
+ maxEphemeralReleaseTombstones = 16384
+)
+
+func effectiveEphemeralCapacity(roots []string, byteLimit, minFreeBytes int64) (int64, int64, error) {
+ if byteLimit > 0 && minFreeBytes > 0 {
+ return byteLimit, minFreeBytes, nil
+ }
+ var smallestTotal int64
+ var largestTotal int64
+ for _, root := range roots {
+ diskInfo, err := xsysinfo.GetDiskInfo(root)
+ if err != nil {
+ return 0, 0, fmt.Errorf("reading ephemeral filesystem capacity for %q: %w", root, err)
+ }
+ total := int64(min(diskInfo.Total, uint64(math.MaxInt64)))
+ if smallestTotal == 0 || total < smallestTotal {
+ smallestTotal = total
+ }
+ largestTotal = max(largestTotal, total)
+ }
+ if smallestTotal == 0 {
+ return 0, 0, fmt.Errorf("at least one ephemeral root is required")
+ }
+ if byteLimit <= 0 {
+ byteLimit = min(defaultEphemeralByteLimitCeiling, smallestTotal/10)
+ }
+ if minFreeBytes <= 0 {
+ minFreeBytes = max(defaultEphemeralMinFreeFloor, largestTotal/20)
+ }
+ return byteLimit, minFreeBytes, nil
+}
+
+// EphemeralCapacityError reports the values used to reject a reservation.
+type EphemeralCapacityError struct {
+ RequestedBytes int64
+ UsageBytes int64
+ LimitBytes int64
+ AvailableBytes int64
+ HeadroomBytes int64
+}
+
+func (e *EphemeralCapacityError) Error() string {
+ return fmt.Sprintf(
+ "ephemeral capacity exceeded: requested=%d usage=%d limit=%d available=%d headroom=%d",
+ e.RequestedBytes,
+ e.UsageBytes,
+ e.LimitBytes,
+ e.AvailableBytes,
+ e.HeadroomBytes,
+ )
+}
+
+// EphemeralReservationConflictError reports an attempt to change an active
+// reservation without first committing or releasing it.
+type EphemeralReservationConflictError struct {
+ Path string
+ ActiveBytes int64
+ RequestedBytes int64
+}
+
+// EphemeralRequestReleasedError reports an attempt to stage another file for
+// a request whose cleanup has already begun.
+type EphemeralRequestReleasedError struct {
+ RequestID string
+}
+
+func (e *EphemeralRequestReleasedError) Error() string {
+ return fmt.Sprintf("ephemeral request %q has already been released", e.RequestID)
+}
+
+type ephemeralReleaseTombstone struct {
+ requestID string
+ expires time.Time
+}
+
+func (e *EphemeralReservationConflictError) Error() string {
+ return fmt.Sprintf(
+ "ephemeral path %q already has an active reservation of %d bytes; requested %d bytes",
+ e.Path,
+ e.ActiveBytes,
+ e.RequestedBytes,
+ )
+}
+
+type ephemeralCapacityState uint8
+
+const (
+ ephemeralCapacityExisting ephemeralCapacityState = iota
+ ephemeralCapacityActive
+ ephemeralCapacityWriting
+ ephemeralCapacityCommitted
+)
+
+type ephemeralCapacityEntry struct {
+ state ephemeralCapacityState
+ owned bool
+ releaseRequested bool
+ baseline int64
+ reserved int64
+ pending int64
+ inflight int64
+ openWriters int
+}
+
+// EphemeralCapacityGuard accounts files and in-flight writes below a fixed set
+// of ephemeral roots.
+type EphemeralCapacityGuard struct {
+ mu sync.Mutex
+ changed *sync.Cond
+ roots []string
+ byteLimit int64
+ minFreeBytes int64
+ usage int64
+ entries map[string]ephemeralCapacityEntry
+ commitWaiters map[string]int
+ released map[string]time.Time
+ releaseOrder []ephemeralReleaseTombstone
+ requestOps map[string]int
+ closingOps map[string]bool
+ releasePins map[string]int
+}
+
+// NewEphemeralCapacityGuard creates a guard and accounts regular files already
+// present below roots. Directory walks do not follow symbolic links.
+func NewEphemeralCapacityGuard(roots []string, byteLimit, minFreeBytes int64) (*EphemeralCapacityGuard, error) {
+ if len(roots) == 0 {
+ return nil, fmt.Errorf("at least one ephemeral root is required")
+ }
+ if byteLimit < 0 {
+ return nil, fmt.Errorf("ephemeral byte limit must not be negative")
+ }
+ if minFreeBytes < 0 {
+ return nil, fmt.Errorf("ephemeral free-space headroom must not be negative")
+ }
+
+ guard := &EphemeralCapacityGuard{
+ roots: make([]string, 0, len(roots)),
+ byteLimit: byteLimit,
+ minFreeBytes: minFreeBytes,
+ entries: make(map[string]ephemeralCapacityEntry),
+ commitWaiters: make(map[string]int),
+ released: make(map[string]time.Time),
+ requestOps: make(map[string]int),
+ closingOps: make(map[string]bool),
+ releasePins: make(map[string]int),
+ }
+ guard.changed = sync.NewCond(&guard.mu)
+ seenRoots := make(map[string]struct{}, len(roots))
+ for _, root := range roots {
+ cleanRoot, err := cleanEphemeralAbsolutePath(root)
+ if err != nil {
+ return nil, fmt.Errorf("resolving ephemeral root: %w", err)
+ }
+ if _, found := seenRoots[cleanRoot]; found {
+ continue
+ }
+ if err := rejectEphemeralSymlinkComponents(cleanRoot); err != nil {
+ return nil, fmt.Errorf("validating ephemeral root %q: %w", cleanRoot, err)
+ }
+ seenRoots[cleanRoot] = struct{}{}
+ guard.roots = append(guard.roots, cleanRoot)
+ }
+
+ for _, root := range guard.roots {
+ if err := guard.accountExistingFiles(root); err != nil {
+ return nil, err
+ }
+ }
+ return guard, nil
+}
+
+// BeginRequestOperation registers staging work before it performs filesystem
+// or object-store operations. A release waits for registered work and prevents
+// new work for the same request from entering.
+func (g *EphemeralCapacityGuard) BeginRequestOperation(requestID string) error {
+ if err := validateEphemeralCacheRequestID(requestID); err != nil {
+ return err
+ }
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ g.pruneReleaseTombstonesLocked(time.Now())
+ if _, found := g.released[requestID]; found || g.closingOps[requestID] {
+ return &EphemeralRequestReleasedError{RequestID: requestID}
+ }
+ if _, found := g.requestOps[requestID]; !found && len(g.requestOps) >= maxEphemeralReleaseTombstones {
+ return fmt.Errorf("too many concurrent ephemeral request operations")
+ }
+ g.requestOps[requestID]++
+ return nil
+}
+
+// EndRequestOperation completes work registered by BeginRequestOperation.
+func (g *EphemeralCapacityGuard) EndRequestOperation(requestID string) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.requestOps[requestID] <= 1 {
+ delete(g.requestOps, requestID)
+ delete(g.closingOps, requestID)
+ } else {
+ g.requestOps[requestID]--
+ }
+ g.pruneReleaseTombstonesLocked(time.Now())
+ g.changed.Broadcast()
+}
+
+// Reserve atomically reserves size additional bytes for path. An existing or
+// committed file remains charged until the new write is committed.
+func (g *EphemeralCapacityGuard) Reserve(path string, size int64) error {
+ if size < 0 {
+ return fmt.Errorf("reservation size must not be negative")
+ }
+ cleanPath, root, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if err := g.rejectReleasedRequestLocked(cleanPath, root); err != nil {
+ return err
+ }
+ entry, found := g.entries[cleanPath]
+ if found && entry.isActive() {
+ if entry.reserved == size {
+ return nil
+ }
+ return &EphemeralReservationConflictError{
+ Path: cleanPath, ActiveBytes: entry.reserved, RequestedBytes: size,
+ }
+ }
+ if err := g.checkCapacityLocked(root, size); err != nil {
+ return err
+ }
+ entry.state = ephemeralCapacityActive
+ entry.owned = true
+ entry.reserved = size
+ entry.pending = size
+ entry.inflight = 0
+ g.entries[cleanPath] = entry
+ g.usage += size
+ return nil
+}
+
+// Commit replaces the path's baseline and reservation with the regular file's
+// actual size. It waits for every bounded writer for the path to close and
+// preserves request ownership until Release.
+func (g *EphemeralCapacityGuard) Commit(path string) error {
+ cleanPath, root, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ for {
+ entry, found := g.entries[cleanPath]
+ if !found {
+ return fmt.Errorf("ephemeral path %q has no reservation", cleanPath)
+ }
+ if entry.state == ephemeralCapacityExisting || entry.state == ephemeralCapacityCommitted {
+ return nil
+ }
+ if entry.openWriters == 0 {
+ break
+ }
+ g.commitWaiters[cleanPath]++
+ g.changed.Wait()
+ g.commitWaiters[cleanPath]--
+ if g.commitWaiters[cleanPath] == 0 {
+ delete(g.commitWaiters, cleanPath)
+ }
+ }
+
+ info, err := os.Lstat(cleanPath)
+ if err != nil {
+ return fmt.Errorf("stating committed ephemeral file %q: %w", cleanPath, err)
+ }
+ if !info.Mode().IsRegular() {
+ return fmt.Errorf("committed ephemeral path %q is not a regular file", cleanPath)
+ }
+ entry := g.entries[cleanPath]
+ charged := entry.baseline + entry.reserved
+ if info.Size() > charged {
+ additional := info.Size() - charged
+ if err := g.checkCapacityLocked(root, additional); err != nil {
+ return err
+ }
+ charged += additional
+ g.usage += additional
+ }
+ g.usage -= charged - info.Size()
+ entry.state = ephemeralCapacityCommitted
+ entry.owned = !entry.releaseRequested && !g.requestReleasedLocked(cleanPath, root)
+ entry.baseline = info.Size()
+ entry.reserved = 0
+ entry.pending = 0
+ entry.inflight = 0
+ g.entries[cleanPath] = entry
+ g.changed.Broadcast()
+ return nil
+}
+
+// Claim makes an existing regular file request-owned until Release. It
+// serializes with recovery deletion, preserves bytes already discovered by a
+// startup scan, and only admits growth that fits the configured capacity.
+// Repeated claims of the same committed file are idempotent.
+func (g *EphemeralCapacityGuard) Claim(path string) error {
+ cleanPath, root, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if err := g.rejectReleasedRequestLocked(cleanPath, root); err != nil {
+ return err
+ }
+
+ info, err := os.Lstat(cleanPath)
+ if err != nil {
+ return fmt.Errorf("stating claimed ephemeral file %q: %w", cleanPath, err)
+ }
+ if !info.Mode().IsRegular() {
+ return fmt.Errorf("claimed ephemeral path %q is not a regular file", cleanPath)
+ }
+
+ entry, found := g.entries[cleanPath]
+ if found && entry.isOwned() && entry.state != ephemeralCapacityCommitted {
+ return &EphemeralReservationConflictError{
+ Path: cleanPath, ActiveBytes: entry.reserved, RequestedBytes: info.Size(),
+ }
+ }
+
+ accounted := int64(0)
+ if found {
+ accounted = entry.baseline + entry.reserved
+ }
+ delta := info.Size() - accounted
+ if delta > 0 {
+ if err := g.checkCapacityLocked(root, delta); err != nil {
+ // The file already occupies the filesystem, so keep accounting
+ // truthful even though a new request cannot claim it. Preserve
+ // ownership if an earlier claim is still awaiting Release.
+ state := ephemeralCapacityExisting
+ owned := false
+ if found && entry.state == ephemeralCapacityCommitted && entry.isOwned() {
+ state = ephemeralCapacityCommitted
+ owned = true
+ }
+ entry = ephemeralCapacityEntry{
+ state: state, owned: owned, baseline: info.Size(),
+ }
+ g.entries[cleanPath] = entry
+ g.usage += delta
+ return err
+ }
+ }
+ g.usage += delta
+ entry = ephemeralCapacityEntry{
+ state: ephemeralCapacityCommitted, owned: true, baseline: info.Size(),
+ }
+ g.entries[cleanPath] = entry
+ return nil
+}
+
+// Release forgets all accounting for path. It is safe to call repeatedly.
+func (g *EphemeralCapacityGuard) Release(path string) error {
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ for {
+ entry, found := g.entries[cleanPath]
+ if !found {
+ return nil
+ }
+ if entry.openWriters == 0 {
+ g.releaseLocked(cleanPath)
+ g.changed.Broadcast()
+ return nil
+ }
+ g.changed.Wait()
+ }
+}
+
+// BeginRequestRelease prevents new staging for requestID and waits until every
+// reservation that started before cleanup has either committed or rolled back.
+// The caller can then enumerate the request directories without missing a file
+// created after the enumeration.
+func (g *EphemeralCapacityGuard) BeginRequestRelease(ctx context.Context, requestID string) error {
+ if ctx == nil {
+ return fmt.Errorf("release context is nil")
+ }
+ if err := validateEphemeralCacheRequestID(requestID); err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ stop := context.AfterFunc(ctx, func() {
+ g.mu.Lock()
+ g.changed.Broadcast()
+ g.mu.Unlock()
+ })
+ defer stop()
+ pinned := false
+ keepPin := false
+ defer func() {
+ if pinned && !keepPin {
+ g.endRequestReleaseLocked(requestID)
+ }
+ g.mu.Unlock()
+ }()
+ for {
+ _, alreadyPinned := g.releasePins[requestID]
+ if alreadyPinned || len(g.releasePins) < maxEphemeralReleaseTombstones {
+ break
+ }
+ if err := ctx.Err(); err != nil {
+ g.makeRequestRecoverableLocked(requestID)
+ return err
+ }
+ g.changed.Wait()
+ }
+ g.releasePins[requestID]++
+ pinned = true
+ now := time.Now()
+ g.pruneReleaseTombstonesLocked(now)
+ if _, found := g.released[requestID]; !found {
+ expires := now.Add(ephemeralReleaseTombstoneTTL)
+ g.released[requestID] = expires
+ g.releaseOrder = append(g.releaseOrder, ephemeralReleaseTombstone{requestID: requestID, expires: expires})
+ g.pruneReleaseTombstonesLocked(now)
+ }
+ for path, entry := range g.entries {
+ _, root, err := g.registeredPathLocked(path)
+ if err == nil && ephemeralRequestID(path, root) == requestID && !entry.isActive() {
+ entry.owned = false
+ g.entries[path] = entry
+ }
+ }
+ for g.requestOps[requestID] > 0 || g.hasActiveRequestLocked(requestID) {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ g.changed.Wait()
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ keepPin = true
+ return nil
+}
+
+// EndRequestRelease allows an inactive release marker to expire or be evicted
+// after the caller has completed its request-directory scan.
+func (g *EphemeralCapacityGuard) EndRequestRelease(requestID string) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ g.endRequestReleaseLocked(requestID)
+}
+
+func (g *EphemeralCapacityGuard) endRequestReleaseLocked(requestID string) {
+ if g.releasePins[requestID] <= 1 {
+ delete(g.releasePins, requestID)
+ } else {
+ g.releasePins[requestID]--
+ }
+ g.pruneReleaseTombstonesLocked(time.Now())
+ g.changed.Broadcast()
+}
+
+// Account records unowned bytes found by recovery after startup. Existing
+// request-owned entries are left unchanged so recovery cannot erase live
+// charges or ownership.
+func (g *EphemeralCapacityGuard) Account(path string, size int64) error {
+ if size < 0 {
+ return fmt.Errorf("accounted size must not be negative")
+ }
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ entry, found := g.entries[cleanPath]
+ if found && entry.isOwned() {
+ return &EphemeralReservationConflictError{
+ Path: cleanPath, ActiveBytes: entry.reserved, RequestedBytes: size,
+ }
+ }
+ newUsage := g.usage
+ if found {
+ newUsage -= entry.baseline
+ }
+ if size > math.MaxInt64-newUsage {
+ return fmt.Errorf("ephemeral usage exceeds supported size")
+ }
+ entry = ephemeralCapacityEntry{state: ephemeralCapacityExisting, baseline: size}
+ g.entries[cleanPath] = entry
+ g.usage = newUsage + size
+ return nil
+}
+
+// ReleaseTree forgets accounting for files at or below path. Recovery cleanup
+// can call this after it successfully removes a stale request tree.
+func (g *EphemeralCapacityGuard) ReleaseTree(path string) error {
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ for g.hasOpenWriterLocked(cleanPath) {
+ g.changed.Wait()
+ }
+ for entryPath := range g.entries {
+ if ephemeralPathAtOrBelow(entryPath, cleanPath) {
+ g.releaseLocked(entryPath)
+ }
+ }
+ return nil
+}
+
+// RemoveTreeIfInactive serializes recovery deletion with new reservations so
+// cleanup cannot remove a request tree between an ownership check and Reserve.
+func (g *EphemeralCapacityGuard) RemoveTreeIfInactive(path string, remove func() error) (bool, error) {
+ if remove == nil {
+ return false, fmt.Errorf("ephemeral tree remover is nil")
+ }
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return false, err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ for entryPath, entry := range g.entries {
+ if entry.isOwned() && ephemeralPathAtOrBelow(entryPath, cleanPath) {
+ return false, nil
+ }
+ }
+ if err := remove(); err != nil {
+ return false, err
+ }
+ for entryPath := range g.entries {
+ if ephemeralPathAtOrBelow(entryPath, cleanPath) {
+ g.releaseLocked(entryPath)
+ }
+ }
+ return true, nil
+}
+
+// HasActiveReservation reports whether path itself or a descendant is owned
+// by a request. Recovery cleanup uses it to avoid live request trees, including
+// inputs whose upload has committed while inference is still running.
+func (g *EphemeralCapacityGuard) HasActiveReservation(path string) bool {
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return false
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ for entryPath, entry := range g.entries {
+ if entry.isOwned() && ephemeralPathAtOrBelow(entryPath, cleanPath) {
+ return true
+ }
+ }
+ return false
+}
+
+func (g *EphemeralCapacityGuard) commitWaiterCount(path string) int {
+ cleanPath, _, err := g.registeredPath(path)
+ if err != nil {
+ return 0
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ return g.commitWaiters[cleanPath]
+}
+
+func (g *EphemeralCapacityGuard) releaseTombstoneCount() int {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ g.pruneReleaseTombstonesLocked(time.Now())
+ return len(g.released)
+}
+
+// NewWriter returns a writer that admits unknown-length input in bounded
+// chunks. Callers must close it before committing or releasing the path.
+func (g *EphemeralCapacityGuard) NewWriter(path string, destination io.Writer) (*EphemeralCapacityWriter, error) {
+ if destination == nil {
+ return nil, fmt.Errorf("ephemeral writer destination is nil")
+ }
+ cleanPath, root, err := g.registeredPath(path)
+ if err != nil {
+ return nil, err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if err := g.rejectReleasedRequestLocked(cleanPath, root); err != nil {
+ return nil, err
+ }
+ entry, found := g.entries[cleanPath]
+ if !found || !entry.isActive() {
+ if err := g.checkCapacityLocked(root, 0); err != nil {
+ return nil, err
+ }
+ entry.state = ephemeralCapacityActive
+ entry.reserved = 0
+ entry.pending = 0
+ entry.inflight = 0
+ }
+ entry.owned = true
+ entry.state = ephemeralCapacityWriting
+ entry.openWriters++
+ g.entries[cleanPath] = entry
+ return &EphemeralCapacityWriter{guard: g, path: cleanPath, destination: destination}, nil
+}
+
+// CapacityWriter exposes NewWriter through the transport-facing interface
+// without leaking the concrete writer type across packages.
+func (g *EphemeralCapacityGuard) CapacityWriter(path string, destination io.Writer) (io.WriteCloser, error) {
+ return g.NewWriter(path, destination)
+}
+
+// EphemeralCapacityWriter bounds writes through an EphemeralCapacityGuard.
+// Close finalizes its accounting lifecycle without closing the destination.
+type EphemeralCapacityWriter struct {
+ mu sync.Mutex
+ guard *EphemeralCapacityGuard
+ path string
+ destination io.Writer
+ closed bool
+}
+
+var _ io.WriteCloser = (*EphemeralCapacityWriter)(nil)
+
+func (w *EphemeralCapacityWriter) Write(payload []byte) (int, error) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return 0, fmt.Errorf("ephemeral capacity writer is closed")
+ }
+
+ total := 0
+ for len(payload) > 0 {
+ chunkLength := min(len(payload), int(ephemeralCapacityWriteChunk))
+ grown, err := w.guard.beginWrite(w.path, int64(chunkLength))
+ if err != nil {
+ return total, err
+ }
+ written, writeErr := w.destination.Write(payload[:chunkLength])
+ if written < 0 || written > chunkLength {
+ w.guard.settleWrite(w.path, int64(chunkLength), 0, grown)
+ return total, fmt.Errorf("ephemeral destination returned invalid write count %d", written)
+ }
+ w.guard.settleWrite(w.path, int64(chunkLength), int64(written), grown)
+ total += written
+ payload = payload[written:]
+ if writeErr != nil {
+ return total, writeErr
+ }
+ if written != chunkLength {
+ return total, io.ErrShortWrite
+ }
+ }
+ return total, nil
+}
+
+func (w *EphemeralCapacityWriter) Close() error {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return nil
+ }
+ w.closed = true
+ w.guard.closeWriter(w.path)
+ return nil
+}
+
+func (e ephemeralCapacityEntry) isActive() bool {
+ return e.state == ephemeralCapacityActive || e.state == ephemeralCapacityWriting
+}
+
+func (e ephemeralCapacityEntry) isOwned() bool {
+ return e.owned
+}
+
+func (g *EphemeralCapacityGuard) accountExistingFiles(root string) error {
+ err := filepath.WalkDir(root, func(path string, dirEntry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if dirEntry.Type()&os.ModeSymlink != 0 {
+ if dirEntry.IsDir() {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ info, err := dirEntry.Info()
+ if err != nil {
+ return err
+ }
+ if !info.Mode().IsRegular() {
+ return nil
+ }
+ cleanPath := filepath.Clean(path)
+ if _, found := g.entries[cleanPath]; found {
+ return nil
+ }
+ if info.Size() > math.MaxInt64-g.usage {
+ return fmt.Errorf("ephemeral usage exceeds supported size")
+ }
+ g.entries[cleanPath] = ephemeralCapacityEntry{
+ state: ephemeralCapacityExisting, baseline: info.Size(),
+ }
+ g.usage += info.Size()
+ return nil
+ })
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("accounting ephemeral root %q: %w", root, err)
+ }
+ return nil
+}
+
+func (g *EphemeralCapacityGuard) beginWrite(path string, size int64) (int64, error) {
+ _, root, err := g.registeredPath(path)
+ if err != nil {
+ return 0, err
+ }
+
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ entry, found := g.entries[path]
+ if !found || entry.state != ephemeralCapacityWriting || entry.openWriters == 0 {
+ return 0, fmt.Errorf("ephemeral path %q has no active writer", path)
+ }
+ availableReservation := entry.pending - entry.inflight
+ grow := max(int64(0), size-availableReservation)
+ if err := g.checkCapacityLocked(root, grow); err != nil {
+ return 0, err
+ }
+ entry.reserved += grow
+ entry.pending += grow
+ entry.inflight += size
+ g.entries[path] = entry
+ g.usage += grow
+ return grow, nil
+}
+
+func (g *EphemeralCapacityGuard) settleWrite(path string, attempted, written, grown int64) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ entry, found := g.entries[path]
+ if !found {
+ return
+ }
+ entry.inflight -= attempted
+ entry.pending -= written
+ unusedGrowth := min(grown, attempted-written)
+ entry.reserved -= unusedGrowth
+ entry.pending -= unusedGrowth
+ g.usage -= unusedGrowth
+ g.entries[path] = entry
+}
+
+func (g *EphemeralCapacityGuard) closeWriter(path string) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ entry, found := g.entries[path]
+ if !found || entry.openWriters == 0 {
+ return
+ }
+ entry.openWriters--
+ if entry.openWriters == 0 {
+ entry.state = ephemeralCapacityActive
+ }
+ g.entries[path] = entry
+ g.changed.Broadcast()
+}
+
+func (g *EphemeralCapacityGuard) checkCapacityLocked(root string, requested int64) error {
+ available, err := ephemeralAvailableBytes(root)
+ if err != nil {
+ return err
+ }
+ exceedsLimit := requested > g.byteLimit-g.usage
+ pending := g.pendingLocked()
+ freeAfterHeadroom := available - min(available, g.minFreeBytes)
+ exceedsAvailable := pending > freeAfterHeadroom || requested > freeAfterHeadroom-pending
+ if exceedsLimit || exceedsAvailable {
+ return &EphemeralCapacityError{
+ RequestedBytes: requested,
+ UsageBytes: g.usage,
+ LimitBytes: g.byteLimit,
+ AvailableBytes: available,
+ HeadroomBytes: g.minFreeBytes,
+ }
+ }
+ return nil
+}
+
+func (g *EphemeralCapacityGuard) pendingLocked() int64 {
+ var pending int64
+ for _, entry := range g.entries {
+ if entry.pending > math.MaxInt64-pending {
+ return math.MaxInt64
+ }
+ pending += entry.pending
+ }
+ return pending
+}
+
+func (g *EphemeralCapacityGuard) hasOpenWriterLocked(path string) bool {
+ for entryPath, entry := range g.entries {
+ if entry.openWriters > 0 && ephemeralPathAtOrBelow(entryPath, path) {
+ return true
+ }
+ }
+ return false
+}
+
+func (g *EphemeralCapacityGuard) releaseLocked(path string) {
+ entry, found := g.entries[path]
+ if !found {
+ return
+ }
+ g.usage -= entry.baseline + entry.reserved
+ delete(g.entries, path)
+}
+
+func (g *EphemeralCapacityGuard) hasActiveRequestLocked(requestID string) bool {
+ for path, entry := range g.entries {
+ if !entry.isActive() {
+ continue
+ }
+ _, root, err := g.registeredPathLocked(path)
+ if err == nil && ephemeralRequestID(path, root) == requestID {
+ return true
+ }
+ }
+ return false
+}
+
+func (g *EphemeralCapacityGuard) rejectReleasedRequestLocked(path, root string) error {
+ requestID := ephemeralRequestID(path, root)
+ if requestID == "" {
+ return nil
+ }
+ g.pruneReleaseTombstonesLocked(time.Now())
+ if _, found := g.released[requestID]; found || g.closingOps[requestID] {
+ return &EphemeralRequestReleasedError{RequestID: requestID}
+ }
+ return nil
+}
+
+func (g *EphemeralCapacityGuard) makeRequestRecoverableLocked(requestID string) {
+ if g.requestOps[requestID] > 0 {
+ g.closingOps[requestID] = true
+ }
+ for path, entry := range g.entries {
+ _, root, err := g.registeredPathLocked(path)
+ if err != nil || ephemeralRequestID(path, root) != requestID {
+ continue
+ }
+ entry.owned = false
+ entry.releaseRequested = true
+ g.entries[path] = entry
+ }
+}
+
+func (g *EphemeralCapacityGuard) requestReleasedLocked(path, root string) bool {
+ requestID := ephemeralRequestID(path, root)
+ if requestID == "" {
+ return false
+ }
+ g.pruneReleaseTombstonesLocked(time.Now())
+ _, found := g.released[requestID]
+ return found
+}
+
+func (g *EphemeralCapacityGuard) pruneReleaseTombstonesLocked(now time.Time) {
+ kept := g.releaseOrder[:0]
+ for _, marker := range g.releaseOrder {
+ expires, found := g.released[marker.requestID]
+ if !found || !expires.Equal(marker.expires) {
+ continue
+ }
+ removable := g.requestOps[marker.requestID] == 0 && g.releasePins[marker.requestID] == 0 &&
+ (!marker.expires.After(now) || len(g.released) > maxEphemeralReleaseTombstones)
+ if removable {
+ delete(g.released, marker.requestID)
+ continue
+ }
+ kept = append(kept, marker)
+ }
+ g.releaseOrder = kept
+}
+
+func ephemeralRequestID(path, root string) string {
+ relative, err := filepath.Rel(root, path)
+ if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
+ return ""
+ }
+ parts := strings.Split(relative, string(filepath.Separator))
+ if len(parts) < 2 {
+ return ""
+ }
+ return parts[1]
+}
+
+func (g *EphemeralCapacityGuard) registeredPath(path string) (string, string, error) {
+ cleanPath, err := cleanEphemeralAbsolutePath(path)
+ if err != nil {
+ return "", "", err
+ }
+ root := ""
+ for _, candidate := range g.roots {
+ if ephemeralPathAtOrBelow(cleanPath, candidate) && len(candidate) > len(root) {
+ root = candidate
+ }
+ }
+ if root == "" {
+ return "", "", fmt.Errorf("path %q is outside registered ephemeral roots", cleanPath)
+ }
+ if err := rejectEphemeralSymlinkComponents(cleanPath); err != nil {
+ return "", "", fmt.Errorf("validating ephemeral path %q: %w", cleanPath, err)
+ }
+ return cleanPath, root, nil
+}
+
+// registeredPathLocked validates a path already stored by the guard. Stored
+// paths were validated on admission, so this avoids filesystem work while the
+// guard mutex is held during request scans.
+func (g *EphemeralCapacityGuard) registeredPathLocked(cleanPath string) (string, string, error) {
+ root := ""
+ for _, candidate := range g.roots {
+ if ephemeralPathAtOrBelow(cleanPath, candidate) && len(candidate) > len(root) {
+ root = candidate
+ }
+ }
+ if root == "" {
+ return "", "", fmt.Errorf("path %q is outside registered ephemeral roots", cleanPath)
+ }
+ return cleanPath, root, nil
+}
+
+func cleanEphemeralAbsolutePath(path string) (string, error) {
+ if path == "" {
+ return "", fmt.Errorf("ephemeral path is empty")
+ }
+ absPath, err := filepath.Abs(path)
+ if err != nil {
+ return "", fmt.Errorf("resolving ephemeral path %q: %w", path, err)
+ }
+ return filepath.Clean(absPath), nil
+}
+
+func rejectEphemeralSymlinkComponents(path string) error {
+ volume := filepath.VolumeName(path)
+ remainder := strings.TrimPrefix(path, volume)
+ current := volume + string(filepath.Separator)
+ remainder = strings.TrimPrefix(remainder, string(filepath.Separator))
+ for _, component := range strings.Split(remainder, string(filepath.Separator)) {
+ if component == "" {
+ continue
+ }
+ current = filepath.Join(current, component)
+ info, err := os.Lstat(current)
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("checking path component %q: %w", current, err)
+ }
+ if info.Mode()&os.ModeSymlink != 0 {
+ return fmt.Errorf("path component %q is a symlink", current)
+ }
+ }
+ return nil
+}
+
+func ephemeralPathAtOrBelow(path, parent string) bool {
+ return path == parent || strings.HasPrefix(path, parent+string(filepath.Separator))
+}
+
+func ephemeralAvailableBytes(path string) (int64, error) {
+ diskInfo, err := xsysinfo.GetDiskInfo(path)
+ if err != nil {
+ return 0, fmt.Errorf("reading ephemeral filesystem availability: %w", err)
+ }
+ if diskInfo.Available > math.MaxInt64 {
+ return math.MaxInt64, nil
+ }
+ return int64(diskInfo.Available), nil
+}
diff --git a/core/services/worker/ephemeral_capacity_test.go b/core/services/worker/ephemeral_capacity_test.go
new file mode 100644
index 000000000..a5b4fd1f4
--- /dev/null
+++ b/core/services/worker/ephemeral_capacity_test.go
@@ -0,0 +1,484 @@
+// SPDX-License-Identifier: MIT
+
+package worker
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type capacityGatedFileWriter struct {
+ file *os.File
+ entered chan struct{}
+ resume chan struct{}
+ once sync.Once
+}
+
+func (w *capacityGatedFileWriter) Write(p []byte) (int, error) {
+ w.once.Do(func() {
+ close(w.entered)
+ <-w.resume
+ })
+ return w.file.Write(p)
+}
+
+type capacityShortWriter struct{}
+
+func (capacityShortWriter) Write(p []byte) (int, error) {
+ if len(p) == 0 {
+ return 0, nil
+ }
+ return 1, nil
+}
+
+var _ = Describe("EphemeralCapacityGuard", func() {
+ It("derives bounded defaults and preserves positive overrides", func() {
+ root := GinkgoT().TempDir()
+ limit, headroom, err := effectiveEphemeralCapacity([]string{root}, 0, -1)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(limit).To(BeNumerically(">", 0))
+ Expect(limit).To(BeNumerically("<=", defaultEphemeralByteLimitCeiling))
+ Expect(headroom).To(BeNumerically(">=", defaultEphemeralMinFreeFloor))
+
+ limit, headroom, err = effectiveEphemeralCapacity([]string{root}, 123, 456)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(limit).To(Equal(int64(123)))
+ Expect(headroom).To(Equal(int64(456)))
+ })
+
+ It("accounts existing regular files without following symlinks", func() {
+ root := GinkgoT().TempDir()
+ outside := filepath.Join(GinkgoT().TempDir(), "outside.bin")
+ Expect(os.WriteFile(filepath.Join(root, "existing.bin"), make([]byte, 6), 0o600)).To(Succeed())
+ Expect(os.WriteFile(outside, make([]byte, 100), 0o600)).To(Succeed())
+ Expect(os.Symlink(outside, filepath.Join(root, "outside-link"))).To(Succeed())
+
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ err = guard.Reserve(filepath.Join(root, "next.bin"), 5)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.RequestedBytes).To(Equal(int64(5)))
+ Expect(capacityErr.UsageBytes).To(Equal(int64(6)))
+ Expect(capacityErr.LimitBytes).To(Equal(int64(10)))
+ Expect(capacityErr.AvailableBytes).To(BeNumerically(">", 0))
+ Expect(capacityErr.HeadroomBytes).To(BeZero())
+ })
+
+ It("serializes competing reservations", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 1, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ start := make(chan struct{})
+ results := make(chan error, 32)
+ var wait sync.WaitGroup
+ for i := range 32 {
+ wait.Add(1)
+ go func(index int) {
+ defer wait.Done()
+ <-start
+ results <- guard.Reserve(filepath.Join(root, string(rune('a'+index))), 1)
+ }(i)
+ }
+ close(start)
+ wait.Wait()
+ close(results)
+
+ succeeded := 0
+ for result := range results {
+ if result == nil {
+ succeeded++
+ continue
+ }
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(result, &capacityErr)).To(BeTrue())
+ }
+ Expect(succeeded).To(Equal(1))
+ })
+
+ It("makes only an equal active reservation idempotent", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "nested", "payload.bin")
+
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "nested", ".", "payload.bin"), 4)).To(Succeed())
+ err = guard.Reserve(path, 5)
+ var conflictErr *EphemeralReservationConflictError
+ Expect(errors.As(err, &conflictErr)).To(BeTrue())
+ Expect(conflictErr.ActiveBytes).To(Equal(int64(4)))
+ Expect(conflictErr.RequestedBytes).To(Equal(int64(5)))
+ Expect(guard.Reserve(filepath.Join(root, "other.bin"), 6)).To(Succeed())
+ Expect(guard.Release(filepath.Join(root, "nested", ".", "payload.bin"))).To(Succeed())
+ Expect(guard.Release(path)).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "replacement.bin"), 4)).To(Succeed())
+ })
+
+ It("retains committed bytes when the same path starts another reservation", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "payload.bin")
+
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ Expect(os.WriteFile(path, make([]byte, 4), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ Expect(guard.HasActiveReservation(path)).To(BeTrue())
+ Expect(guard.Reserve(path, 6)).To(Succeed())
+ Expect(guard.HasActiveReservation(path)).To(BeTrue())
+
+ err = guard.Reserve(filepath.Join(root, "overflow.bin"), 1)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.UsageBytes).To(Equal(int64(10)))
+ })
+
+ It("retains startup-accounted bytes when the path is reserved", func() {
+ root := GinkgoT().TempDir()
+ path := filepath.Join(root, "payload.bin")
+ Expect(os.WriteFile(path, make([]byte, 4), 0o600)).To(Succeed())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+
+ Expect(guard.Reserve(path, 6)).To(Succeed())
+ Expect(guard.HasActiveReservation(path)).To(BeTrue())
+ err = guard.Reserve(filepath.Join(root, "overflow.bin"), 1)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.UsageBytes).To(Equal(int64(10)))
+ })
+
+ It("commits the regular file's actual size", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "payload.bin")
+
+ Expect(guard.Reserve(path, 10)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("four"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ Expect(guard.Commit(filepath.Clean(path))).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "six.bin"), 6)).To(Succeed())
+ })
+
+ It("preserves configured filesystem headroom", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 1<<30, 1<<62)
+ Expect(err).NotTo(HaveOccurred())
+
+ err = guard.Reserve(filepath.Join(root, "payload.bin"), 1)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.RequestedBytes).To(Equal(int64(1)))
+ Expect(capacityErr.AvailableBytes).To(BeNumerically(">", 0))
+ Expect(capacityErr.HeadroomBytes).To(Equal(int64(1 << 62)))
+ })
+
+ It("reserves bounded chunks before forwarding unknown-length input", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, ephemeralCapacityWriteChunk+1, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "payload.bin")
+ file, err := os.Create(path)
+ Expect(err).NotTo(HaveOccurred())
+ DeferCleanup(file.Close)
+ writer, err := guard.NewWriter(path, file)
+ Expect(err).NotTo(HaveOccurred())
+
+ n, err := writer.Write(make([]byte, ephemeralCapacityWriteChunk+2))
+ Expect(n).To(Equal(int(ephemeralCapacityWriteChunk)))
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(writer.Close()).To(Succeed())
+ info, err := file.Stat()
+ Expect(err).NotTo(HaveOccurred())
+ Expect(info.Size()).To(Equal(ephemeralCapacityWriteChunk))
+ })
+
+ It("waits for an open bounded writer before committing", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "payload.bin")
+ file, err := os.Create(path)
+ Expect(err).NotTo(HaveOccurred())
+ DeferCleanup(file.Close)
+ gated := &capacityGatedFileWriter{
+ file: file, entered: make(chan struct{}), resume: make(chan struct{}),
+ }
+ DeferCleanup(func() {
+ select {
+ case <-gated.resume:
+ default:
+ close(gated.resume)
+ }
+ })
+ writer, err := guard.NewWriter(path, gated)
+ Expect(err).NotTo(HaveOccurred())
+
+ writeDone := make(chan error, 1)
+ go func() {
+ _, writeErr := writer.Write([]byte("1234567"))
+ writeDone <- writeErr
+ }()
+ Eventually(gated.entered).Should(BeClosed())
+ commitDone := make(chan error, 1)
+ go func() { commitDone <- guard.Commit(path) }()
+ Eventually(func() int { return guard.commitWaiterCount(path) }).Should(Equal(1))
+ Expect(commitDone).NotTo(Receive())
+
+ close(gated.resume)
+ Expect(<-writeDone).To(Succeed())
+ Expect(commitDone).NotTo(Receive())
+ Expect(writer.Close()).To(Succeed())
+ Eventually(commitDone).Should(Receive(Succeed()))
+
+ err = guard.Reserve(filepath.Join(root, "other.bin"), 4)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.UsageBytes).To(Equal(int64(7)))
+ })
+
+ It("does not share pending capacity between concurrent writers", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "payload.bin")
+ file, err := os.Create(path)
+ Expect(err).NotTo(HaveOccurred())
+ DeferCleanup(file.Close)
+ gated := &capacityGatedFileWriter{
+ file: file, entered: make(chan struct{}), resume: make(chan struct{}),
+ }
+ first, err := guard.NewWriter(path, gated)
+ Expect(err).NotTo(HaveOccurred())
+ var secondDestination bytes.Buffer
+ second, err := guard.NewWriter(path, &secondDestination)
+ Expect(err).NotTo(HaveOccurred())
+
+ firstDone := make(chan error, 1)
+ go func() {
+ _, writeErr := first.Write([]byte("1234567"))
+ firstDone <- writeErr
+ }()
+ Eventually(gated.entered).Should(BeClosed())
+
+ n, err := second.Write([]byte("7654321"))
+ Expect(n).To(BeZero())
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(secondDestination.Len()).To(BeZero())
+
+ close(gated.resume)
+ Expect(<-firstDone).To(Succeed())
+ Expect(first.Close()).To(Succeed())
+ Expect(second.Close()).To(Succeed())
+ })
+
+ It("rolls back bytes the destination writer does not accept", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 5, 0)
+ Expect(err).NotTo(HaveOccurred())
+ writer, err := guard.NewWriter(filepath.Join(root, "payload.bin"), capacityShortWriter{})
+ Expect(err).NotTo(HaveOccurred())
+
+ n, err := writer.Write([]byte("123"))
+ Expect(n).To(Equal(1))
+ Expect(err).To(MatchError(io.ErrShortWrite))
+ Expect(writer.Close()).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "other.bin"), 4)).To(Succeed())
+ })
+
+ It("rejects paths outside roots and through symlinks", func() {
+ root := GinkgoT().TempDir()
+ outside := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 100, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ Expect(guard.Reserve(filepath.Join(outside, "payload.bin"), 1)).To(
+ MatchError(ContainSubstring("outside registered ephemeral roots")),
+ )
+ Expect(os.Symlink(outside, filepath.Join(root, "escape"))).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "escape", "payload.bin"), 1)).To(
+ MatchError(ContainSubstring("symlink")),
+ )
+ })
+
+ It("supports recovery tree accounting without dropping active reservations", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ active := filepath.Join(root, "active", "payload.bin")
+ stale := filepath.Join(root, "stale", "payload.bin")
+
+ Expect(guard.Reserve(active, 4)).To(Succeed())
+ Expect(guard.Account(stale, 3)).To(Succeed())
+ Expect(guard.HasActiveReservation(root)).To(BeTrue())
+ Expect(guard.ReleaseTree(filepath.Join(root, "stale"))).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "replacement.bin"), 6)).To(Succeed())
+ Expect(guard.ReleaseTree(root)).To(Succeed())
+ Expect(guard.HasActiveReservation(root)).To(BeFalse())
+ })
+
+ It("waits for pre-release reservations before request cleanup scans", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "audio", "request-1", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+
+ released := make(chan error, 1)
+ go func() {
+ released <- guard.BeginRequestRelease(context.Background(), "request-1")
+ }()
+ Eventually(guard.releaseTombstoneCount).Should(Equal(1))
+ Consistently(released, 50*time.Millisecond).ShouldNot(Receive())
+
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ Eventually(released).Should(Receive(Succeed()))
+ guard.EndRequestRelease("request-1")
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+ })
+
+ It("rejects staging after request cleanup begins", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(guard.BeginRequestRelease(context.Background(), "request-1")).To(Succeed())
+ defer guard.EndRequestRelease("request-1")
+
+ err = guard.Reserve(filepath.Join(root, "audio", "request-1", "late.wav"), 1)
+ var releasedErr *EphemeralRequestReleasedError
+ Expect(errors.As(err, &releasedErr)).To(BeTrue())
+ Expect(releasedErr.RequestID).To(Equal("request-1"))
+ })
+
+ It("leaves a late commit recoverable when release times out", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "audio", "request-1", "late.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ Expect(guard.BeginRequestRelease(ctx, "request-1")).To(MatchError(context.Canceled))
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+ })
+
+ It("bounds release markers without reopening registered work", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(guard.BeginRequestOperation("request-pinned")).To(Succeed())
+ pinnedRelease := make(chan error, 1)
+ go func() {
+ pinnedRelease <- guard.BeginRequestRelease(context.Background(), "request-pinned")
+ }()
+ Eventually(guard.releaseTombstoneCount).Should(Equal(1))
+ guard.EndRequestOperation("request-pinned")
+ Eventually(pinnedRelease).Should(Receive(Succeed()))
+
+ for index := range maxEphemeralReleaseTombstones + 10 {
+ requestID := fmt.Sprintf("request-%d", index)
+ Expect(guard.BeginRequestRelease(context.Background(), requestID)).To(Succeed())
+ guard.EndRequestRelease(requestID)
+ }
+ Expect(guard.releaseTombstoneCount()).To(Equal(maxEphemeralReleaseTombstones))
+ err = guard.Reserve(filepath.Join(root, "audio", "request-pinned", "late.wav"), 1)
+ var releasedErr *EphemeralRequestReleasedError
+ Expect(errors.As(err, &releasedErr)).To(BeTrue())
+ guard.EndRequestRelease("request-pinned")
+ })
+
+ It("applies backpressure at the release-pin cap and clears ownership", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "audio", "request-target", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+
+ guard.mu.Lock()
+ for index := range maxEphemeralReleaseTombstones {
+ guard.releasePins[fmt.Sprintf("pinned-%d", index)] = 1
+ }
+ guard.mu.Unlock()
+ released := make(chan error, 1)
+ go func() {
+ released <- guard.BeginRequestRelease(context.Background(), "request-target")
+ }()
+ Consistently(released, 50*time.Millisecond).ShouldNot(Receive())
+
+ guard.EndRequestRelease("pinned-0")
+ Eventually(released).Should(Receive(Succeed()))
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+ guard.EndRequestRelease("request-target")
+ })
+
+ It("makes committed files recoverable when pin backpressure expires", func() {
+ root := GinkgoT().TempDir()
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ path := filepath.Join(root, "audio", "request-target", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ guard.mu.Lock()
+ for index := range maxEphemeralReleaseTombstones {
+ guard.releasePins[fmt.Sprintf("pinned-%d", index)] = 1
+ }
+ guard.mu.Unlock()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ Expect(guard.BeginRequestRelease(ctx, "request-target")).To(MatchError(context.Canceled))
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+ })
+
+ It("rejects a registered cache-hit claim after pin backpressure expires", func() {
+ root := GinkgoT().TempDir()
+ path := filepath.Join(root, "audio", "request-target", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(guard.BeginRequestOperation("request-target")).To(Succeed())
+ defer guard.EndRequestOperation("request-target")
+ guard.mu.Lock()
+ for index := range maxEphemeralReleaseTombstones {
+ guard.releasePins[fmt.Sprintf("pinned-%d", index)] = 1
+ }
+ guard.mu.Unlock()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ Expect(guard.BeginRequestRelease(ctx, "request-target")).To(MatchError(context.Canceled))
+ err = guard.Claim(path)
+ var releasedErr *EphemeralRequestReleasedError
+ Expect(errors.As(err, &releasedErr)).To(BeTrue())
+ Expect(guard.HasActiveReservation(path)).To(BeFalse())
+ })
+})
diff --git a/core/services/worker/ephemeral_cleanup.go b/core/services/worker/ephemeral_cleanup.go
index 4a8e92da9..4f3fe243a 100644
--- a/core/services/worker/ephemeral_cleanup.go
+++ b/core/services/worker/ephemeral_cleanup.go
@@ -14,9 +14,9 @@ const (
// outlive the request that needed it. Inference reads these files while the
// request runs, so the window has to cover a slow multimodal request; it
// does not have to cover anything longer.
- defaultEphemeralStagingTTL = 6 * time.Hour
+ defaultEphemeralStagingTTL = time.Hour
// defaultEphemeralStagingSweep is how often the worker sweeps.
- defaultEphemeralStagingSweep = 30 * time.Minute
+ defaultEphemeralStagingSweep = 15 * time.Minute
)
// StartEphemeralStagingCleanup sweeps the worker's own staging directory for
@@ -32,6 +32,15 @@ func StartEphemeralStagingCleanup(ctx context.Context, stagingDir string, ttl, i
if stagingDir == "" {
return
}
+ StartEphemeralRootsCleanup(ctx, []string{filepath.Join(stagingDir, "ephemeral")}, nil, ttl, interval)
+}
+
+// StartEphemeralRootsCleanup removes abandoned request inputs for every worker
+// transport while sharing accounting with live reservations.
+func StartEphemeralRootsCleanup(ctx context.Context, roots []string, guard *EphemeralCapacityGuard, ttl, interval time.Duration) {
+ if len(roots) == 0 {
+ return
+ }
if ttl <= 0 {
ttl = defaultEphemeralStagingTTL
}
@@ -39,31 +48,41 @@ func StartEphemeralStagingCleanup(ctx context.Context, stagingDir string, ttl, i
interval = defaultEphemeralStagingSweep
}
+ // Reclaim crash leftovers before the caller starts accepting new work.
+ CleanEphemeralRoots(roots, ttl, guard)
+
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
- // Sweep once at startup: a worker that crashed with staged files leaves
- // them behind, and waiting a full interval to reclaim that space is the
- // case that hurts on a volume that is already close to full.
- CleanEphemeralStaging(stagingDir, ttl)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
- CleanEphemeralStaging(stagingDir, ttl)
+ CleanEphemeralRoots(roots, ttl, guard)
}
}
}()
- xlog.Info("Ephemeral staging cleanup started", "dir", stagingDir, "ttl", ttl, "interval", interval)
+ xlog.Info("Ephemeral staging cleanup started", "roots", roots, "ttl", ttl, "interval", interval)
}
// CleanEphemeralStaging removes staged per-request directories older than ttl.
// It only ever descends into /ephemeral, so staged model weights,
// which live alongside it and are not scratch, are never considered.
func CleanEphemeralStaging(stagingDir string, ttl time.Duration) {
- root := filepath.Join(stagingDir, "ephemeral")
+ CleanEphemeralRoots([]string{filepath.Join(stagingDir, "ephemeral")}, ttl, nil)
+}
+
+// CleanEphemeralRoots removes stale request directories from explicit
+// ephemeral roots. WalkDir never follows directory symlinks.
+func CleanEphemeralRoots(roots []string, ttl time.Duration, guard *EphemeralCapacityGuard) {
+ for _, root := range roots {
+ cleanEphemeralRoot(root, ttl, guard)
+ }
+}
+
+func cleanEphemeralRoot(root string, ttl time.Duration, guard *EphemeralCapacityGuard) {
categories, err := os.ReadDir(root)
if err != nil {
// A worker that has never served a file-bearing request has no
@@ -87,19 +106,31 @@ func CleanEphemeralStaging(stagingDir string, ttl time.Duration) {
continue
}
for _, entry := range entries {
- path := filepath.Join(categoryDir, entry.Name())
- info, err := entry.Info()
+ if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ requestPath := filepath.Join(categoryDir, entry.Name())
+ newest, err := newestEphemeralModTime(requestPath)
if err != nil {
- xlog.Warn("Ephemeral staging cleanup: cannot stat entry", "path", path, "error", err)
+ xlog.Warn("Ephemeral staging cleanup: cannot inspect request", "path", requestPath, "error", err)
continue
}
- // A request rewrites nothing after staging, so the entry's own
- // modification time is when its request was served.
- if !info.ModTime().Before(cutoff) {
+ if !newest.Before(cutoff) {
continue
}
- if err := os.RemoveAll(path); err != nil {
- xlog.Warn("Ephemeral staging cleanup: cannot remove", "path", path, "error", err)
+ if guard != nil {
+ removedTree, err := guard.RemoveTreeIfInactive(requestPath, func() error {
+ return os.RemoveAll(requestPath)
+ })
+ if err != nil {
+ xlog.Warn("Ephemeral staging cleanup: cannot remove", "path", requestPath, "error", err)
+ continue
+ }
+ if !removedTree {
+ continue
+ }
+ } else if err := os.RemoveAll(requestPath); err != nil {
+ xlog.Warn("Ephemeral staging cleanup: cannot remove", "path", requestPath, "error", err)
continue
}
removed++
@@ -110,3 +141,24 @@ func CleanEphemeralStaging(stagingDir string, ttl time.Duration) {
xlog.Info("Ephemeral staging cleanup removed stale request files", "count", removed, "dir", root)
}
}
+
+func newestEphemeralModTime(root string) (time.Time, error) {
+ var newest time.Time
+ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ info, err := entry.Info()
+ if err != nil {
+ return err
+ }
+ if info.ModTime().After(newest) {
+ newest = info.ModTime()
+ }
+ if entry.Type()&os.ModeSymlink != 0 && entry.IsDir() {
+ return filepath.SkipDir
+ }
+ return nil
+ })
+ return newest, err
+}
diff --git a/core/services/worker/ephemeral_cleanup_test.go b/core/services/worker/ephemeral_cleanup_test.go
index 3542f1afc..c8cb9e150 100644
--- a/core/services/worker/ephemeral_cleanup_test.go
+++ b/core/services/worker/ephemeral_cleanup_test.go
@@ -55,4 +55,60 @@ var _ = Describe("Worker ephemeral staging cleanup", func() {
It("does nothing when no ephemeral directory exists", func() {
Expect(func() { CleanEphemeralStaging(stagingDir, time.Hour) }).ToNot(Panic())
})
+
+ It("sweeps both transport roots by newest descendant and skips active requests", func() {
+ cacheDir := GinkgoT().TempDir()
+ httpRoot := filepath.Join(stagingDir, "ephemeral")
+ s3Root := filepath.Join(cacheDir, "ephemeral")
+ guard, err := NewEphemeralCapacityGuard([]string{httpRoot, s3Root}, 8, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ staleRequest := filepath.Join(httpRoot, "audio", "stale")
+ activeRequest := filepath.Join(s3Root, "audio", "active")
+ freshChildRequest := filepath.Join(s3Root, "audio", "fresh-child")
+ for _, requestDir := range []string{staleRequest, activeRequest, freshChildRequest} {
+ Expect(os.MkdirAll(requestDir, 0o750)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(requestDir, "input.bin"), []byte("data"), 0o600)).To(Succeed())
+ }
+ old := time.Now().Add(-2 * time.Hour)
+ fresh := time.Now().Add(-5 * time.Minute)
+ for _, requestDir := range []string{staleRequest, activeRequest, freshChildRequest} {
+ Expect(os.Chtimes(requestDir, old, old)).To(Succeed())
+ }
+ Expect(os.Chtimes(filepath.Join(staleRequest, "input.bin"), old, old)).To(Succeed())
+ Expect(os.Chtimes(filepath.Join(activeRequest, "input.bin"), old, old)).To(Succeed())
+ Expect(os.Chtimes(filepath.Join(freshChildRequest, "input.bin"), fresh, fresh)).To(Succeed())
+ Expect(guard.Account(filepath.Join(staleRequest, "input.bin"), 4)).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(activeRequest, "input.bin"), 4)).To(Succeed())
+
+ CleanEphemeralRoots([]string{httpRoot, s3Root}, time.Hour, guard)
+
+ Expect(staleRequest).NotTo(BeADirectory())
+ Expect(activeRequest).To(BeADirectory())
+ Expect(freshChildRequest).To(BeADirectory())
+ Expect(guard.Reserve(filepath.Join(httpRoot, "audio", "replacement", "input.bin"), 4)).To(Succeed())
+ })
+
+ It("keeps committed request inputs until exact release ends ownership", func() {
+ root := filepath.Join(stagingDir, "ephemeral")
+ requestDir := filepath.Join(root, "audio", "owned")
+ path := filepath.Join(requestDir, "input.bin")
+ Expect(os.MkdirAll(requestDir, 0o750)).To(Succeed())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 8, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ Expect(guard.Reserve(path, 4)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())
+ Expect(guard.Commit(path)).To(Succeed())
+ old := time.Now().Add(-2 * time.Hour)
+ Expect(os.Chtimes(path, old, old)).To(Succeed())
+ Expect(os.Chtimes(requestDir, old, old)).To(Succeed())
+
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(requestDir).To(BeADirectory())
+
+ Expect(guard.Release(path)).To(Succeed())
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(requestDir).NotTo(BeADirectory())
+ })
})
diff --git a/core/services/worker/file_staging.go b/core/services/worker/file_staging.go
index 019afcba9..3eea3a9e6 100644
--- a/core/services/worker/file_staging.go
+++ b/core/services/worker/file_staging.go
@@ -3,14 +3,19 @@ package worker
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"os"
+ "path"
"path/filepath"
"strings"
+ "time"
"github.com/mudler/LocalAI/core/services/messaging"
"github.com/mudler/LocalAI/core/services/storage"
+ "github.com/mudler/LocalAI/pkg/safefile"
"github.com/mudler/xlog"
+ "golang.org/x/sync/singleflight"
)
// isPathAllowed checks if path is within one of the allowed directories.
@@ -37,7 +42,7 @@ func isPathAllowed(path string, allowedDirs []string) bool {
}
// subscribeFileStaging subscribes to NATS file staging subjects for this node.
-func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, nodeID string) error {
+func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, nodeID string, capacity *EphemeralCapacityGuard) error {
// Create FileManager with same S3 config as the frontend
// TODO: propagate a caller-provided context once Config carries one
s3Store, err := storage.NewS3Store(context.Background(), storage.S3Config{
@@ -57,6 +62,10 @@ func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, no
if err != nil {
return fmt.Errorf("initializing file manager: %w", err)
}
+ if err := subscribeFileReleaseWithCapacity(natsClient, nodeID, fm, cacheDir, capacity); err != nil {
+ return err
+ }
+ var ensureGroup singleflight.Group
// Subscribe: files.ensure — download S3 key to local, reply with local path
if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesEnsure(nodeID), func(data []byte, reply func([]byte)) {
@@ -68,12 +77,19 @@ func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, no
return
}
- localPath, err := fm.Download(context.Background(), req.Key)
+ value, err, _ := ensureGroup.Do(req.Key, func() (any, error) {
+ return ensureWorkerFile(context.Background(), fm, capacity, req.Key)
+ })
if err != nil {
xlog.Error("File ensure failed", "key", req.Key, "error", err)
replyJSON(reply, map[string]string{"error": err.Error()})
return
}
+ localPath, ok := value.(string)
+ if !ok {
+ replyJSON(reply, map[string]string{"error": fmt.Sprintf("unexpected file ensure result %T", value)})
+ return
+ }
xlog.Debug("File ensured locally", "key", req.Key, "path", localPath)
replyJSON(reply, map[string]string{"local_path": localPath})
@@ -199,3 +215,220 @@ func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, no
xlog.Info("Subscribed to file staging NATS subjects", "nodeID", nodeID)
return nil
}
+
+func subscribeFileRelease(natsClient messaging.MessagingClient, nodeID string, fm *storage.FileManager, cacheDir string) error {
+ return subscribeFileReleaseWithCapacity(natsClient, nodeID, fm, cacheDir, nil)
+}
+
+func subscribeFileReleaseWithCapacity(natsClient messaging.MessagingClient, nodeID string, fm *storage.FileManager, cacheDir string, capacity *EphemeralCapacityGuard) error {
+ if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesRelease(nodeID), func(data []byte, reply func([]byte)) {
+ var req struct {
+ Key string `json:"key"`
+ RequestID string `json:"request_id"`
+ }
+ if err := json.Unmarshal(data, &req); err != nil {
+ replyJSON(reply, map[string]string{"error": "invalid request"})
+ return
+ }
+ var err error
+ if req.RequestID != "" {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ err = releaseEphemeralCacheRequest(ctx, cacheDir, req.RequestID, capacity)
+ cancel()
+ } else {
+ cachePath, cacheErr := fm.CachePath(req.Key)
+ err = cacheErr
+ if err == nil {
+ err = releaseEphemeralCachePathWithCapacity(cacheDir, req.Key, cachePath, capacity)
+ }
+ }
+ if err != nil {
+ replyJSON(reply, map[string]string{"error": err.Error()})
+ return
+ }
+ replyJSON(reply, map[string]string{})
+ }); err != nil {
+ return fmt.Errorf("subscribing to files.release events: %w", err)
+ }
+ return nil
+}
+
+func releaseEphemeralCacheKey(cacheDir, key string) error {
+ return releaseEphemeralCachePath(cacheDir, key, filepath.Join(cacheDir, filepath.FromSlash(key)))
+}
+
+func releaseEphemeralCachePath(cacheDir, key, filePath string) error {
+ return releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, nil)
+}
+
+func releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath string, capacity *EphemeralCapacityGuard) error {
+ if err := validateEphemeralCacheKey(key); err != nil {
+ return err
+ }
+ relativePath := filepath.FromSlash(key)
+ expectedPath := filepath.Join(cacheDir, relativePath)
+ if filepath.Clean(filePath) != expectedPath {
+ return fmt.Errorf("release path %q does not match key %q", filePath, key)
+ }
+ if err := safefile.RemoveExact(cacheDir, relativePath, []string{".sha256", ".sha256.target"}, 2); err != nil {
+ return err
+ }
+ for _, path := range []string{filePath, filePath + ".sha256", filePath + ".sha256.target"} {
+ if capacity != nil {
+ if err := capacity.Release(path); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func releaseEphemeralCacheRequest(ctx context.Context, cacheDir, requestID string, capacity *EphemeralCapacityGuard) error {
+ if err := validateEphemeralCacheRequestID(requestID); err != nil {
+ return err
+ }
+ if capacity != nil {
+ if err := capacity.BeginRequestRelease(ctx, requestID); err != nil {
+ return fmt.Errorf("beginning release for request %q: %w", requestID, err)
+ }
+ defer capacity.EndRequestRelease(requestID)
+ }
+ root := filepath.Join(cacheDir, "ephemeral")
+ categories, err := os.ReadDir(root)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ var releaseErrors []error
+ for _, category := range categories {
+ if !category.IsDir() || category.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ requestDir := filepath.Join(root, category.Name(), requestID)
+ info, err := os.Lstat(requestDir)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ releaseErrors = append(releaseErrors, fmt.Errorf("stating request directory %q: %w", requestDir, err))
+ }
+ continue
+ }
+ if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
+ releaseErrors = append(releaseErrors, fmt.Errorf("ephemeral request path %q is not a real directory", requestDir))
+ continue
+ }
+ entries, err := os.ReadDir(requestDir)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ releaseErrors = append(releaseErrors, fmt.Errorf("reading request directory %q: %w", requestDir, err))
+ }
+ continue
+ }
+ for _, entry := range entries {
+ if entry.IsDir() {
+ releaseErrors = append(releaseErrors, fmt.Errorf("unexpected directory in ephemeral request %q", filepath.Join(requestDir, entry.Name())))
+ continue
+ }
+ key := filepath.ToSlash(filepath.Join("ephemeral", category.Name(), requestID, entry.Name()))
+ filePath := filepath.Join(cacheDir, filepath.FromSlash(key))
+ if err := releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, capacity); err != nil {
+ releaseErrors = append(releaseErrors, fmt.Errorf("releasing %q: %w", key, err))
+ }
+ }
+ }
+ return errors.Join(releaseErrors...)
+}
+
+type ephemeralStagingCapacity interface {
+ Reserve(path string, size int64) error
+ Commit(path string) error
+ Claim(path string) error
+ Release(path string) error
+}
+
+func ensureWorkerFile(ctx context.Context, fm *storage.FileManager, capacity *EphemeralCapacityGuard, key string) (string, error) {
+ if capacity == nil {
+ return fm.Download(ctx, key)
+ }
+ if strings.HasPrefix(key, "ephemeral/") {
+ if err := validateEphemeralCacheKey(key); err != nil {
+ return "", err
+ }
+ requestID := strings.Split(key, "/")[2]
+ if err := capacity.BeginRequestOperation(requestID); err != nil {
+ return "", err
+ }
+ defer capacity.EndRequestOperation(requestID)
+ }
+ return ensureWorkerFileWithCapacity(ctx, fm, capacity, key)
+}
+
+func ensureWorkerFileWithCapacity(ctx context.Context, fm *storage.FileManager, capacity ephemeralStagingCapacity, key string) (string, error) {
+ if capacity == nil || !strings.HasPrefix(key, "ephemeral/") {
+ return fm.Download(ctx, key)
+ }
+ if err := validateEphemeralCacheKey(key); err != nil {
+ return "", err
+ }
+ cachePath, err := fm.CachePath(key)
+ if err != nil {
+ return "", err
+ }
+ if info, statErr := os.Lstat(cachePath); statErr == nil {
+ if !info.Mode().IsRegular() {
+ return "", fmt.Errorf("ephemeral cache path %q is not a regular file", cachePath)
+ }
+ if err := capacity.Claim(cachePath); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ return "", err
+ }
+ } else {
+ return cachePath, nil
+ }
+ } else if !os.IsNotExist(statErr) {
+ return "", statErr
+ }
+
+ meta, err := fm.Head(ctx, key)
+ if err != nil {
+ return "", fmt.Errorf("reading size for %s: %w", key, err)
+ }
+ if err := capacity.Reserve(cachePath, meta.Size); err != nil {
+ return "", err
+ }
+ localPath, err := fm.Download(ctx, key)
+ if err != nil {
+ _ = capacity.Release(cachePath)
+ return "", err
+ }
+ if err := capacity.Commit(cachePath); err != nil {
+ _ = fm.EvictCache(key)
+ _ = capacity.Release(cachePath)
+ return "", err
+ }
+ return localPath, nil
+}
+
+func validateEphemeralCacheKey(key string) error {
+ if strings.Contains(key, "\\") || path.Clean(key) != key {
+ return fmt.Errorf("invalid ephemeral key %q", key)
+ }
+ parts := strings.Split(key, "/")
+ if len(parts) != 4 || parts[0] != "ephemeral" {
+ return fmt.Errorf("release key %q must identify one file below ephemeral/", key)
+ }
+ for _, part := range parts[1:] {
+ if part == "" || part == "." || part == ".." {
+ return fmt.Errorf("invalid ephemeral key %q", key)
+ }
+ }
+ return nil
+}
+
+func validateEphemeralCacheRequestID(requestID string) error {
+ if requestID == "" || strings.ContainsAny(requestID, "/\\") || path.Clean(requestID) != requestID || requestID == "." || requestID == ".." {
+ return fmt.Errorf("invalid ephemeral request ID %q", requestID)
+ }
+ return nil
+}
diff --git a/core/services/worker/file_staging_release_test.go b/core/services/worker/file_staging_release_test.go
new file mode 100644
index 000000000..62dfcd3cb
--- /dev/null
+++ b/core/services/worker/file_staging_release_test.go
@@ -0,0 +1,405 @@
+package worker
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/mudler/LocalAI/core/services/messaging"
+ "github.com/mudler/LocalAI/core/services/nodes"
+ "github.com/mudler/LocalAI/core/services/storage"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type stagingObjectStore struct {
+ payload []byte
+ getCalls int
+ getErr error
+}
+
+type disappearingStagingCapacity struct{}
+
+func (*disappearingStagingCapacity) Reserve(string, int64) error { return nil }
+func (*disappearingStagingCapacity) Commit(string) error { return nil }
+func (*disappearingStagingCapacity) Release(string) error { return nil }
+func (*disappearingStagingCapacity) Claim(path string) error {
+ if err := os.Remove(path); err != nil {
+ return err
+ }
+ return fmt.Errorf("claim raced recovery: %w", os.ErrNotExist)
+}
+
+func (*stagingObjectStore) Put(context.Context, string, io.Reader) error { return nil }
+func (s *stagingObjectStore) Get(context.Context, string) (io.ReadCloser, error) {
+ s.getCalls++
+ if s.getErr != nil {
+ return nil, s.getErr
+ }
+ return io.NopCloser(strings.NewReader(string(s.payload))), nil
+}
+func (s *stagingObjectStore) Head(_ context.Context, key string) (*storage.ObjectMeta, error) {
+ return &storage.ObjectMeta{Key: key, Size: int64(len(s.payload))}, nil
+}
+func (*stagingObjectStore) Exists(context.Context, string) (bool, error) { return true, nil }
+func (*stagingObjectStore) Delete(context.Context, string) error { return nil }
+func (*stagingObjectStore) List(context.Context, string) ([]string, error) {
+ return nil, nil
+}
+
+type releaseSubscription struct{}
+
+func (releaseSubscription) Unsubscribe() error { return nil }
+
+type releaseMessagingClient struct {
+ subject string
+ handler func([]byte, func([]byte))
+}
+
+func (m *releaseMessagingClient) Publish(string, any) error { return nil }
+func (m *releaseMessagingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) {
+ return releaseSubscription{}, nil
+}
+func (m *releaseMessagingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) {
+ return releaseSubscription{}, nil
+}
+func (m *releaseMessagingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) {
+ return releaseSubscription{}, nil
+}
+func (m *releaseMessagingClient) SubscribeReply(subject string, handler func([]byte, func([]byte))) (messaging.Subscription, error) {
+ m.subject = subject
+ m.handler = handler
+ return releaseSubscription{}, nil
+}
+func (m *releaseMessagingClient) Request(string, []byte, time.Duration) ([]byte, error) {
+ return nil, nil
+}
+func (m *releaseMessagingClient) IsConnected() bool { return true }
+func (m *releaseMessagingClient) Close() {}
+
+var _ = Describe("Worker exact-key staging release", func() {
+ It("protects a startup-accounted HTTP cache hit through authenticated repeated probes", func() {
+ stagingDir := GinkgoT().TempDir()
+ root := filepath.Join(stagingDir, "ephemeral")
+ key := "ephemeral/audio/request-id/input.wav"
+ remotePath := filepath.Join(stagingDir, filepath.FromSlash(key))
+ content := []byte("data")
+ Expect(os.MkdirAll(filepath.Dir(remotePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(remotePath, content, 0o600)).To(Succeed())
+ hash := sha256.Sum256(content)
+ Expect(os.WriteFile(remotePath+".sha256", []byte(fmt.Sprintf("%x", hash)), 0o600)).To(Succeed())
+ old := time.Now().Add(-2 * time.Hour)
+ for _, path := range []string{remotePath, remotePath + ".sha256", filepath.Dir(remotePath)} {
+ Expect(os.Chtimes(path, old, old)).To(Succeed())
+ }
+ guard, err := NewEphemeralCapacityGuard([]string{root}, int64(len(content)+sha256.Size*2), 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ Expect(err).NotTo(HaveOccurred())
+ addr := listener.Addr().String()
+ Expect(listener.Close()).To(Succeed())
+ server, err := nodes.StartFileTransferServerWithCapacity(addr, stagingDir, GinkgoT().TempDir(), GinkgoT().TempDir(), "secret", 0, nil, guard)
+ Expect(err).NotTo(HaveOccurred())
+ DeferCleanup(nodes.ShutdownFileTransferServer, server)
+
+ localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+ Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
+ stager := nodes.NewHTTPFileStager(func(string) (string, error) { return addr, nil }, "secret")
+ for range 2 {
+ path, ensureErr := stager.EnsureRemote(context.Background(), "worker", localPath, key)
+ Expect(ensureErr).NotTo(HaveOccurred())
+ Expect(path).To(Equal(remotePath))
+ }
+
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(remotePath).To(BeAnExistingFile())
+
+ Expect(stager.ReleaseRemote(context.Background(), "worker", key)).To(Succeed())
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(remotePath).NotTo(BeAnExistingFile())
+ })
+
+ It("claims a startup-scanned cache hit against stale recovery until release", func() {
+ cacheDir := GinkgoT().TempDir()
+ root := filepath.Join(cacheDir, "ephemeral")
+ key := "ephemeral/audio/request-id/input.wav"
+ cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(cachePath, []byte("data"), 0o600)).To(Succeed())
+ old := time.Now().Add(-2 * time.Hour)
+ Expect(os.Chtimes(cachePath, old, old)).To(Succeed())
+ Expect(os.Chtimes(filepath.Dir(cachePath), old, old)).To(Succeed())
+
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+ store := &stagingObjectStore{payload: []byte("unused")}
+ fm, err := storage.NewFileManager(store, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+
+ localPath, err := ensureWorkerFile(context.Background(), fm, guard, key)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(localPath).To(Equal(cachePath))
+ Expect(store.getCalls).To(BeZero())
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(cachePath).To(BeAnExistingFile())
+
+ Expect(guard.Release(cachePath)).To(Succeed())
+ CleanEphemeralRoots([]string{root}, time.Hour, guard)
+ Expect(cachePath).NotTo(BeAnExistingFile())
+ })
+
+ It("downloads again when a cache file disappears while being claimed", func() {
+ cacheDir := GinkgoT().TempDir()
+ key := "ephemeral/audio/request-id/input.wav"
+ cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(cachePath, []byte("stale"), 0o600)).To(Succeed())
+ store := &stagingObjectStore{payload: []byte("fresh")}
+ fm, err := storage.NewFileManager(store, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+
+ localPath, err := ensureWorkerFileWithCapacity(context.Background(), fm, &disappearingStagingCapacity{}, key)
+
+ Expect(err).NotTo(HaveOccurred())
+ Expect(localPath).To(Equal(cachePath))
+ Expect(os.ReadFile(localPath)).To(Equal([]byte("fresh")))
+ Expect(store.getCalls).To(Equal(1))
+ })
+
+ It("makes repeated cache-hit claims idempotent", func() {
+ cacheDir := GinkgoT().TempDir()
+ root := filepath.Join(cacheDir, "ephemeral")
+ key := "ephemeral/audio/request-id/input.wav"
+ cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(cachePath, []byte("data"), 0o600)).To(Succeed())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+ fm, err := storage.NewFileManager(&stagingObjectStore{}, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+
+ for range 2 {
+ localPath, ensureErr := ensureWorkerFile(context.Background(), fm, guard, key)
+ Expect(ensureErr).NotTo(HaveOccurred())
+ Expect(localPath).To(Equal(cachePath))
+ }
+ err = guard.Reserve(filepath.Join(root, "other", "request-id", "input.wav"), 1)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.UsageBytes).To(Equal(int64(4)))
+ })
+
+ It("capacity-checks growth of a startup-scanned cache file", func() {
+ cacheDir := GinkgoT().TempDir()
+ root := filepath.Join(cacheDir, "ephemeral")
+ key := "ephemeral/audio/request-id/input.wav"
+ cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
+ Expect(os.WriteFile(cachePath, []byte("12"), 0o600)).To(Succeed())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(os.WriteFile(cachePath, []byte("12345"), 0o600)).To(Succeed())
+ fm, err := storage.NewFileManager(&stagingObjectStore{}, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+
+ _, err = ensureWorkerFile(context.Background(), fm, guard, key)
+ var capacityErr *EphemeralCapacityError
+ Expect(errors.As(err, &capacityErr)).To(BeTrue())
+ Expect(capacityErr.RequestedBytes).To(Equal(int64(3)))
+ Expect(capacityErr.UsageBytes).To(Equal(int64(2)))
+ Expect(guard.HasActiveReservation(cachePath)).To(BeFalse())
+ })
+
+ It("reserves S3 object size before download and releases it with the exact key", func() {
+ cacheDir := GinkgoT().TempDir()
+ root := filepath.Join(cacheDir, "ephemeral")
+ store := &stagingObjectStore{payload: []byte("data")}
+ fm, err := storage.NewFileManager(store, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+ key := "ephemeral/audio/request-id/input.wav"
+
+ localPath, err := ensureWorkerFile(context.Background(), fm, guard, key)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(localPath).To(BeAnExistingFile())
+ Expect(store.getCalls).To(Equal(1))
+ Expect(guard.Reserve(filepath.Join(root, "audio", "other", "input.wav"), 1)).NotTo(Succeed())
+
+ Expect(releaseEphemeralCachePathWithCapacity(cacheDir, key, localPath, guard)).To(Succeed())
+ Expect(guard.Reserve(filepath.Join(root, "audio", "other", "input.wav"), 4)).To(Succeed())
+ })
+
+ It("rejects an oversized S3 object before starting its download", func() {
+ cacheDir := GinkgoT().TempDir()
+ store := &stagingObjectStore{payload: []byte("oversized")}
+ fm, err := storage.NewFileManager(store, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ guard, err := NewEphemeralCapacityGuard([]string{filepath.Join(cacheDir, "ephemeral")}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ _, err = ensureWorkerFile(context.Background(), fm, guard, "ephemeral/audio/request-id/input.wav")
+ Expect(err).To(HaveOccurred())
+ Expect(store.getCalls).To(BeZero())
+ })
+
+ It("rolls back an S3 reservation when the download fails", func() {
+ cacheDir := GinkgoT().TempDir()
+ root := filepath.Join(cacheDir, "ephemeral")
+ store := &stagingObjectStore{payload: []byte("data"), getErr: errors.New("download failed")}
+ fm, err := storage.NewFileManager(store, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ guard, err := NewEphemeralCapacityGuard([]string{root}, 4, 0)
+ Expect(err).NotTo(HaveOccurred())
+
+ _, err = ensureWorkerFile(context.Background(), fm, guard, "ephemeral/audio/request-id/input.wav")
+ Expect(err).To(MatchError(ContainSubstring("download failed")))
+ Expect(guard.Reserve(filepath.Join(root, "audio", "replacement", "input.wav"), 4)).To(Succeed())
+ })
+
+ It("removes only the exact cache file and upload sidecars", func() {
+ cacheDir := GinkgoT().TempDir()
+ categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
+ Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
+ target := filepath.Join(categoryDir, "input.wav")
+ sibling := filepath.Join(categoryDir, "keep.wav")
+ for _, path := range []string{target, target + ".sha256", target + ".sha256.target", sibling} {
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+ }
+
+ Expect(releaseEphemeralCacheKey(cacheDir, "ephemeral/request-id/audio/input.wav")).To(Succeed())
+ Expect(target).NotTo(BeAnExistingFile())
+ Expect(target + ".sha256").NotTo(BeAnExistingFile())
+ Expect(target + ".sha256.target").NotTo(BeAnExistingFile())
+ Expect(sibling).To(BeAnExistingFile())
+ Expect(categoryDir).To(BeADirectory())
+ })
+
+ It("succeeds for a missing file and prunes empty category and request directories", func() {
+ cacheDir := GinkgoT().TempDir()
+ categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
+ Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
+
+ for range 2 {
+ Expect(releaseEphemeralCacheKey(cacheDir, "ephemeral/request-id/audio/missing.wav")).To(Succeed())
+ }
+ Expect(categoryDir).NotTo(BeADirectory())
+ Expect(filepath.Dir(categoryDir)).NotTo(BeADirectory())
+ Expect(filepath.Join(cacheDir, "ephemeral")).To(BeADirectory())
+ })
+
+ It("rejects traversal and symlink escapes", func() {
+ cacheDir := GinkgoT().TempDir()
+ outsideDir := GinkgoT().TempDir()
+ outsidePath := filepath.Join(outsideDir, "input.wav")
+ Expect(os.WriteFile(outsidePath, []byte("keep"), 0640)).To(Succeed())
+ requestDir := filepath.Join(cacheDir, "ephemeral", "request-id")
+ Expect(os.MkdirAll(requestDir, 0750)).To(Succeed())
+ Expect(os.Symlink(outsideDir, filepath.Join(requestDir, "audio"))).To(Succeed())
+
+ for _, key := range []string{
+ "models/model.gguf",
+ "ephemeral/../models/model.gguf",
+ "ephemeral/request-id/audio/../../model.gguf",
+ "ephemeral/request-id/audio/input.wav",
+ } {
+ Expect(releaseEphemeralCacheKey(cacheDir, key)).NotTo(Succeed(), key)
+ }
+ Expect(outsidePath).To(BeAnExistingFile())
+ })
+
+ It("rejects symlinked files and sidecars without deleting their targets", func() {
+ for _, linkedName := range []string{"input.wav", "input.wav.sha256", "input.wav.sha256.target"} {
+ cacheDir := GinkgoT().TempDir()
+ categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
+ Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
+ target := filepath.Join(categoryDir, "input.wav")
+ if linkedName != "input.wav" {
+ Expect(os.WriteFile(target, []byte("input"), 0640)).To(Succeed())
+ }
+ preserved := filepath.Join(cacheDir, "ephemeral", "preserved-"+linkedName)
+ Expect(os.WriteFile(preserved, []byte("keep"), 0640)).To(Succeed())
+ Expect(os.Symlink(preserved, filepath.Join(categoryDir, linkedName))).To(Succeed())
+
+ Expect(releaseEphemeralCacheKey(cacheDir, "ephemeral/request-id/audio/input.wav")).NotTo(Succeed(), linkedName)
+ Expect(preserved).To(BeAnExistingFile(), linkedName)
+ }
+ })
+
+ It("registers an exact release handler", func() {
+ cacheDir := GinkgoT().TempDir()
+ path := filepath.Join(cacheDir, "ephemeral", "request-id", "audio", "input.wav")
+ Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+ fm, err := storage.NewFileManager(nil, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ client := &releaseMessagingClient{}
+
+ Expect(subscribeFileRelease(client, "node.one", fm, cacheDir)).To(Succeed())
+ Expect(client.subject).To(Equal(messaging.SubjectNodeFilesRelease("node.one")))
+ request, err := json.Marshal(map[string]string{"key": "ephemeral/request-id/audio/input.wav"})
+ Expect(err).NotTo(HaveOccurred())
+ var response []byte
+ client.handler(request, func(data []byte) { response = append([]byte(nil), data...) })
+
+ var reply map[string]string
+ Expect(json.Unmarshal(response, &reply)).To(Succeed())
+ Expect(reply["error"]).To(BeEmpty())
+ Expect(path).NotTo(BeAnExistingFile())
+ })
+
+ It("releases a request batch through one worker message", func() {
+ cacheDir := GinkgoT().TempDir()
+ keys := []string{
+ "ephemeral/audio/request-id/input.wav",
+ "ephemeral/images/request-id/frame.jpg",
+ }
+ for _, key := range keys {
+ path := filepath.Join(cacheDir, filepath.FromSlash(key))
+ Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+ Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
+ }
+ fm, err := storage.NewFileManager(nil, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ client := &releaseMessagingClient{}
+ Expect(subscribeFileRelease(client, "node.one", fm, cacheDir)).To(Succeed())
+ request, err := json.Marshal(map[string]any{"request_id": "request-id"})
+ Expect(err).NotTo(HaveOccurred())
+ var response []byte
+
+ client.handler(request, func(data []byte) { response = append([]byte(nil), data...) })
+
+ var reply map[string]string
+ Expect(json.Unmarshal(response, &reply)).To(Succeed())
+ Expect(reply["error"]).To(BeEmpty())
+ for _, key := range keys {
+ Expect(filepath.Join(cacheDir, filepath.FromSlash(key))).NotTo(BeAnExistingFile())
+ }
+ })
+
+ It("returns validation errors through the release handler", func() {
+ cacheDir := GinkgoT().TempDir()
+ fm, err := storage.NewFileManager(nil, cacheDir)
+ Expect(err).NotTo(HaveOccurred())
+ client := &releaseMessagingClient{}
+ Expect(subscribeFileRelease(client, "node-1", fm, cacheDir)).To(Succeed())
+
+ request, err := json.Marshal(map[string]string{"key": "models/model.gguf"})
+ Expect(err).NotTo(HaveOccurred())
+ var response []byte
+ client.handler(request, func(data []byte) { response = append([]byte(nil), data...) })
+
+ var reply map[string]string
+ Expect(json.Unmarshal(response, &reply)).To(Succeed())
+ Expect(reply["error"]).NotTo(BeEmpty())
+ })
+})
diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go
index 7217b2ad4..2c80167da 100644
--- a/core/services/worker/supervisor.go
+++ b/core/services/worker/supervisor.go
@@ -597,6 +597,7 @@ func (s *backendSupervisor) reapDeadProcess(key string, bp *backendProcess) {
if bp == nil {
return
}
+ s.cleanupProcessRuntime(bp.proc)
if bp.port <= 0 {
xlog.Error("Cannot recycle backend port: dead process has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
return
@@ -614,6 +615,7 @@ func (s *backendSupervisor) releaseBackendStart(key string, bp *backendProcess)
return
}
delete(s.processes, key)
+ s.cleanupProcessRuntime(bp.proc)
if bp.port <= 0 {
xlog.Error("Cannot recycle backend port: startup has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
return
@@ -947,6 +949,7 @@ func (s *backendSupervisor) finishBackendStop(key string, bp *backendProcess, st
return fmt.Errorf("stopping backend process %s: %w", key, stopErr)
}
delete(s.processes, key)
+ s.cleanupProcessRuntime(bp.proc)
if bp.port <= 0 {
xlog.Error("Cannot recycle backend port: process has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
return nil
@@ -955,6 +958,14 @@ func (s *backendSupervisor) finishBackendStop(key string, bp *backendProcess, st
return nil
}
+func (s *backendSupervisor) cleanupProcessRuntime(proc *process.Process) {
+ // Some focused supervisor tests provide synthetic process handles without a
+ // ModelLoader. Production processes always come from s.ml.StartProcess.
+ if s.ml != nil {
+ s.ml.CleanupProcessRuntime(proc)
+ }
+}
+
// stopAllBackends stops all running backend processes and returns the process
// keys it attempted, so a caller answering a backend.stop request can report
// what it acted on.
diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go
index f57a5a4ae..a74e04ff7 100644
--- a/core/services/worker/worker.go
+++ b/core/services/worker/worker.go
@@ -149,22 +149,37 @@ func Run(ctx *cliContext.Context, cfg *Config) error {
// the top of Run so the worker fails before registering.)
httpAddr := cfg.resolveHTTPAddr()
stagingDir := filepath.Join(cfg.ModelsPath, "..", "staging")
+ cacheDir := filepath.Join(cfg.ModelsPath, "..", "cache")
dataDir := filepath.Join(cfg.ModelsPath, "..", "data")
+ ephemeralRoots := []string{
+ filepath.Join(stagingDir, "ephemeral"),
+ filepath.Join(cacheDir, "ephemeral"),
+ }
+ byteLimit, minFreeBytes, err := effectiveEphemeralCapacity(
+ ephemeralRoots,
+ cfg.EphemeralStagingByteLimit,
+ cfg.EphemeralStagingMinFreeBytes,
+ )
+ if err != nil {
+ return fmt.Errorf("resolving ephemeral staging capacity: %w", err)
+ }
+ ephemeralCapacity, err := NewEphemeralCapacityGuard(ephemeralRoots, byteLimit, minFreeBytes)
+ if err != nil {
+ return fmt.Errorf("initializing ephemeral staging capacity: %w", err)
+ }
+ xlog.Info("Ephemeral staging capacity configured", "roots", ephemeralRoots, "byteLimit", byteLimit, "minFreeBytes", minFreeBytes)
+ StartEphemeralRootsCleanup(shutdownCtx, ephemeralRoots, ephemeralCapacity, 0, 0)
// The readiness gate is created here but only armed once NATS is up and the
// backend supervisor exists, below, because the gate probes both.
// Until then /readyz reports ready, which is correct: reaching this line
// means the worker has already registered with the frontend, so it is
// mid-startup rather than broken.
readiness := &nodes.WorkerReadiness{}
- httpServer, err := nodes.StartFileTransferServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ml.BackendLogs())
+ httpServer, err := nodes.StartFileTransferServerWithCapacity(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ephemeralCapacity, ml.BackendLogs())
if err != nil {
return fmt.Errorf("starting HTTP file transfer server: %w", err)
}
- // Per-request input files land in stagingDir over that server and nothing
- // used to remove them, so a long-lived worker filled its own disk.
- StartEphemeralStagingCleanup(shutdownCtx, stagingDir, 0, 0)
-
// Connect to NATS
xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL))
natsClient, err := connectNats()
@@ -249,7 +264,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error {
// Subscribe to file staging NATS subjects if S3 is configured
if cfg.StorageURL != "" {
- if err := cfg.subscribeFileStaging(natsClient, nodeID); err != nil {
+ if err := cfg.subscribeFileStaging(natsClient, nodeID, ephemeralCapacity); err != nil {
nodes.ShutdownFileTransferServer(httpServer)
return fmt.Errorf("subscribing to file staging subjects: %w", err)
}
diff --git a/docs/content/advanced/_index.en.md b/docs/content/advanced/_index.en.md
index c81602ea1..8b0b4ece3 100644
--- a/docs/content/advanced/_index.en.md
+++ b/docs/content/advanced/_index.en.md
@@ -103,8 +103,7 @@ Before diving into advanced topics, ensure you have:
## Related Sections
- 📚 [Reference](../reference/) - API documentation and command reference
-- 🔌 [Installation](../installation/) - Deployment options and requirements
-- ⭐ [Features](../features/) - Overview of LocalAI capabilities
+- - ⭐ [Features](../features/) - Overview of LocalAI capabilities
---
diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md
index 8e0ca855b..d629e1be8 100644
--- a/docs/content/advanced/model-configuration.md
+++ b/docs/content/advanced/model-configuration.md
@@ -397,7 +397,9 @@ The canonical names match upstream llama.cpp (dash-separated). For backward comp
Multiple types can be chained by passing a comma-separated list to `spec_type` (e.g. `spec_type:ngram-simple,ngram-mod`). The runtime tries them in order and accepts the first proposal that meets the acceptance criteria.
{{% notice note %}}
-Speculative decoding is automatically disabled when multimodal models (with `mmproj`) are active. The `n_draft` parameter can also be overridden per-request.
+The current LocalAI llama.cpp backend supports speculative decoding with multimodal models that load an `mmproj`, including MTP. LocalAI passes both configurations to llama.cpp and does not disable speculation merely because an `mmproj` is present. Upstream llama.cpp removed the former general multimodal/speculative restriction in [ggml-org/llama.cpp#19493](https://github.com/ggml-org/llama.cpp/pull/19493); [ggml-org/llama.cpp#22673](https://github.com/ggml-org/llama.cpp/pull/22673) later added MTP support and explicitly documented its compatibility with vision input.
+
+Compatibility still depends on the installed backend version and the target/draft model architecture. Check the backend logs for successful projector loading and speculative-context initialization, then look for the `draft acceptance` statistics line and its `accepted / generated` counts. A representative run with zero accepted draft tokens receives no speculative speedup and can indicate that the model or settings need tuning.
{{% /notice %}}
##### Multi-Token Prediction (MTP)
@@ -427,7 +429,7 @@ Detection runs both at **import time** (the `/import-model` UI / `POST /models/i
| `spec_type` | `draft-mtp` | Activates MTP. Can be chained with other types (see below). |
| `spec_n_max` / `draft_max` | `2`-`6` | Number of draft tokens per step. Upstream's PR suggests 2-3 for the tightest acceptance window; LocalAI's auto-default is 6 to favour throughput on models with high acceptance. |
| `spec_p_min` | `0.75` | Pinned because upstream marks the current default with a "change to 0.0f" TODO; locking it here keeps acceptance thresholds stable across future llama.cpp bumps. |
-| `mmproj_use_gpu` | `false` (or unset `mmproj`) | MTP has a prompt-processing overhead; if the model is non-vision, drop the mmproj entirely to save VRAM. |
+| `mmproj_use_gpu` | `true` for vision | MTP does not require disabling the projector. Keep `mmproj` configured for image input; set this option to `false` to keep the projector on CPU when VRAM is tight. Remove `mmproj` only for text-only use when vision is not needed. |
**Minimal config** (override-only, since auto-detection already covers this for MTP-capable GGUFs):
@@ -441,6 +443,23 @@ options:
- spec_n_max:3
```
+**With vision enabled:**
+
+```yaml
+name: qwen3-vision-mtp
+backend: llama-cpp
+known_usecases:
+ - chat
+ - vision
+parameters:
+ model: qwen3-with-mtp.gguf
+mmproj: mmproj-qwen3.gguf
+options:
+ - spec_type:draft-mtp
+ - spec_n_max:3
+ - spec_p_min:0.75
+```
+
**With a separate MTP head file:**
```yaml
@@ -720,7 +739,7 @@ For image generation models using the `diffusers` backend:
| Field | Type | Description |
|-------|------|-------------|
-| `diffusers.cuda` | bool | Enable CUDA for diffusers |
+| `diffusers.cuda` | bool | Force CUDA. By default the backend auto-detects and uses CUDA when a compatible GPU is present (ROCm builds included). Pin the CPU with `options: ["device:cpu"]` |
| `diffusers.pipeline_type` | string | Pipeline type (e.g., `stable-diffusion`, `stable-diffusion-xl`) |
| `diffusers.scheduler_type` | string | Scheduler type (e.g., `euler`, `ddpm`) |
| `diffusers.enable_parameters` | string | Comma-separated parameters to enable |
diff --git a/docs/content/features/agents.md b/docs/content/features/agents.md
index f406d0d15..cde92862e 100644
--- a/docs/content/features/agents.md
+++ b/docs/content/features/agents.md
@@ -288,6 +288,9 @@ All agent endpoints are grouped under `/api/agents/`:
| `POST` | `/api/agents/collections/:name/upload` | Upload a document |
| `GET` | `/api/agents/collections/:name/entries` | List entries |
| `POST` | `/api/agents/collections/:name/search` | Search a collection |
+| `GET` | `/api/agents/collections/:name/sources` | List external sources |
+| `POST` | `/api/agents/collections/:name/sources` | Add an external source (`update_interval` is an integer number of minutes; defaults to 60) |
+| `DELETE` | `/api/agents/collections/:name/sources` | Remove an external source |
| `POST` | `/api/agents/collections/:name/reset` | Reset a collection |
### Actions
diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md
index 045074340..09759ab72 100644
--- a/docs/content/features/audio-classification.md
+++ b/docs/content/features/audio-classification.md
@@ -11,6 +11,10 @@ LocalAI exposes this through the `/v1/audio/classification` endpoint, modelled a
Because classification is exposed as a regular OpenAI-style endpoint, any HTTP client works - there is no Python dependency on the consumer side.
+In distributed mode, LocalAI stages uploaded audio and realtime sound-detection
+windows on the selected worker before classification. The API server and worker
+do not need a shared temporary directory.
+
## Endpoint
```
diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md
index ae71cd384..9d59cb378 100644
--- a/docs/content/features/audio-cpp.md
+++ b/docs/content/features/audio-cpp.md
@@ -28,6 +28,11 @@ key stored *inside* the GGUF, which cannot be read from a remote repository, and
upstream GGUF repository hosts every family in one place. Set `backend: audio-cpp`
in the model YAML, or select it explicitly in the import form.
+The bundled audio-cpp gallery entries set `backend:best` in `options` to select
+an available compute backend, with CPU as the fallback. To force CPU execution,
+replace that option with `backend:cpu`. Model configurations that omit this
+option still default to CPU.
+
## What it serves
One model serves one family, and a family advertises the tasks it can perform. The
diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md
index 0312d392a..5a5e833cf 100644
--- a/docs/content/features/audio-to-text.md
+++ b/docs/content/features/audio-to-text.md
@@ -11,6 +11,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s
- **[whisper.cpp](https://github.com/ggerganov/whisper.cpp)**: A C++ library for audio transcription (default)
- **moonshine**: Ultra-fast transcription engine optimized for low-end devices
- **faster-whisper**: Fast Whisper implementation with CTranslate2
+- **WhisperX**: Whisper transcription with word alignment and optional speaker diarization. Set `HF_TOKEN` and pass `diarize=true` to load WhisperX's gated pyannote diarization pipeline.
- **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events).
- **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder.
- **voxtral**: Voxtral-family models served by a dedicated backend
@@ -109,7 +110,7 @@ In addition to `file` and `model`, the endpoint accepts the following multipart
| `timestamp_granularities[]` | Multi-value form field: `word` and/or `segment`. Honored when the backend produces the requested granularity. |
| `response_format` | One of `json` (default for backwards-compat), `verbose_json`, `text`, `srt`, `vtt`, `lrc`. |
| `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. |
-| `diarize` | LocalAI extension - speaker diarization (whisper.cpp only). |
+| `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. |
The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled).
diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md
index 16eb01c77..022437bb9 100644
--- a/docs/content/features/distributed-mode.md
+++ b/docs/content/features/distributed-mode.md
@@ -297,6 +297,8 @@ local-ai worker \
| `--advertise-addr` | `LOCALAI_ADVERTISE_ADDR` | *(auto)* | Address the frontend uses to reach this node (see below) |
| `--http-addr` | `LOCALAI_HTTP_ADDR` | gRPC port - 1 | HTTP file transfer server bind address |
| `--advertise-http-addr` | `LOCALAI_ADVERTISE_HTTP_ADDR` | *(auto)* | HTTP address the frontend uses for file transfer |
+| `--ephemeral-staging-byte-limit` | `LOCALAI_EPHEMERAL_STAGING_BYTE_LIMIT` | `0` (automatic) | Maximum bytes held by request-input staging across the worker's HTTP staging directory and S3 cache. Automatic mode uses the smaller of 10 GiB and 10% of filesystem capacity. |
+| `--ephemeral-staging-min-free-bytes` | `LOCALAI_EPHEMERAL_STAGING_MIN_FREE_BYTES` | `0` (automatic) | Free filesystem space preserved while staging request inputs. Automatic mode uses the larger of 1 GiB and 5% of filesystem capacity. |
| `--register-to` | `LOCALAI_REGISTER_TO` | *(required)* | Frontend URL for self-registration |
| `--node-name` | `LOCALAI_NODE_NAME` | hostname | Human-readable node name |
| `--registration-token` | `LOCALAI_REGISTRATION_TOKEN` | *(empty)* | Token to authenticate with the frontend |
@@ -320,6 +322,12 @@ local-ai worker \
**HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). By default it listens on the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. Set `--advertise-http-addr` if the auto-detected address is not routable from the frontend.
{{% /notice %}}
+### Ephemeral request-input storage
+
+Workers reserve local capacity before accepting per-request audio, image, and other ephemeral inputs. The limit covers both direct HTTP staging and the worker's S3 download cache. A request is rejected before inference when accepting its input would exceed the byte limit or the configured free-space headroom. One request-scoped cleanup operation releases all exact input keys and their reservations after inference, while a one-hour recovery sweep removes abandoned files after crashes. The sweep runs at startup and every 15 minutes, preserves active requests, and considers the newest file in each request directory.
+
+Set both capacity variables to positive byte counts when a worker needs fixed limits. Leaving either value at zero selects its filesystem-based default. These settings apply only below the two `ephemeral` roots; model, data, and configuration files are excluded.
+
### Worker Health Probes
The worker's HTTP server (base port - 1, default 50050) exposes two unauthenticated probes:
@@ -1205,9 +1213,17 @@ Notes:
- Verify `--heartbeat-interval` is not set too high
- Offline nodes automatically restore to healthy when they re-register (no re-approval needed)
+**InsightFace reports a missing MiniFASNet file after staging:**
+- Gallery models such as `insightface-buffalo-m` use a virtual primary name and load their files through options. The frontend derives the worker's model directory from successfully staged companion files or directories, so relative options resolve inside the model's staging directory.
+- If logs show matching hashes for the staged files but InsightFace still reports a bare filename such as `MiniFASNetV2.onnx` as missing, upgrade the frontend to include this path-resolution fix. Re-uploading the same files does not correct the directory passed to the backend.
+
**Backend not installing:**
- Check the worker logs for `backend.install` events
+**Model staging repeatedly fails with HTTP 416 after all bytes have arrived:**
+- An interrupted upload can leave a full-size file marked as unfinished (`.sha256.target`). On retry, the worker verifies the file's SHA-256 and finalizes it if it matches, without rewriting the model. Corrupt content fails integrity validation and is removed.
+- Upgrade the affected worker to get this recovery behavior. Older workers can repeatedly reject retries from byte zero with `Content-Range start 0 does not match current file size`. File size alone is not proof that an upload is valid.
+
**Requests still report an old context size or another old load option:**
- Query `/api/nodes/:id/models` for every worker that hosts the model.
- Confirm that every routable replica has `state: loaded` and the same current `config_revision`.
@@ -1230,8 +1246,9 @@ Notes:
- Check the worker process is running and its NATS connection is up. `Scheduled node is not answering on the bus` in the frontend log names each node demoted this way.
**A worker fills its own disk over time:**
-- A request that carries a file (an image, an audio clip, a video) stages that file to the worker under `/../staging/ephemeral/`. The worker deletes these 6 hours after the request that needed them, and sweeps every 30 minutes plus once at startup, so a worker that crashed mid-request still reclaims the space.
-- Releases before this sweep existed kept every staged input for the lifetime of the worker. Delete `/../staging/ephemeral/` on an affected worker once, as the user the worker runs as; the sweep keeps it bounded from then on.
+- A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it.
+- A one-hour recovery sweep runs at startup and every 15 minutes to reclaim inputs left by interrupted requests. It preserves active reservations and uses the newest file timestamp in each request directory.
+- Releases before request-owned cleanup existed can leave a legacy backlog. Delete the affected `ephemeral/` directory once, as the user the worker runs as; capacity admission and recovery cleanup keep new staging bounded.
- Staged **model** files are not touched by this. They live beside the ephemeral directory and are not per-request scratch.
- A worker whose volume is genuinely full reports `creating backend process state directory under ...: no space left on device` when a backend starts.
diff --git a/docs/content/features/face-recognition.md b/docs/content/features/face-recognition.md
index 8b037eec7..33fa29fc4 100644
--- a/docs/content/features/face-recognition.md
+++ b/docs/content/features/face-recognition.md
@@ -73,9 +73,9 @@ Detect faces and analyze demographics (buffalo entries populate
age / gender; YuNet + SFace returns regions only):
```bash
-curl -sX POST http://localhost:8080/v1/face/detect \
+curl -sX POST http://localhost:8080/v1/detection \
-H "Content-Type: application/json" \
- -d '{"model": "face-detect-buffalo-l", "img": "https://example.com/group.jpg"}'
+ -d '{"model": "face-detect-buffalo-l", "image": "https://example.com/group.jpg"}'
curl -sX POST http://localhost:8080/v1/face/analyze \
-H "Content-Type: application/json" \
@@ -141,6 +141,39 @@ Response:
}
```
+## Restore enrollments after a restart
+
+The default identity store is in memory. Clients can keep an enrollment record
+and replay it with `POST /v1/face/register` after a restart. Extract the embedding
+once with `/v1/face/embed`, then save the exact returned vector, model, name,
+labels, and enrollment timestamp. Submit `embedding` instead of `img`:
+
+```json
+{
+ "model": "insightface-opencv",
+ "name": "Alice",
+ "embedding": [0.12, -0.04, 0.31],
+ "registered_at": "2026-09-07T12:00:00Z",
+ "labels": {"client_id": "alice"}
+}
+```
+
+The vector above is abbreviated; send the complete embedding from the same
+recognizer model. Provide exactly one of `img` or `embedding`. Vectors must be
+finite and nonzero. `registered_at` is optional and defaults to the current time;
+replay the original timestamp to preserve it.
+
+The store upserts by exact vector. Registration now derives a stable ID from
+that vector and the store namespace, so retries and replay after a restart
+return the same ID without adding duplicate entries. Replaying updates the name
+and labels. Images can produce slightly different embeddings across runs; keep
+the original vector instead of embedding the photo again on each retry.
+
+This does not make the server store persistent. Clients must retain and restore
+the records themselves. With independent stores behind a load balancer, replay
+into each store or use a shared store. Do not mix different recognizer models in
+one store. IDs from older versions change on their first registration replay.
+
## 1:N identification workflow (register → identify → forget)
This is the primary "face recognition" flow. Under the hood it uses
diff --git a/docs/content/features/image-generation.md b/docs/content/features/image-generation.md
index d94c0f9b4..1df6058b4 100644
--- a/docs/content/features/image-generation.md
+++ b/docs/content/features/image-generation.md
@@ -79,8 +79,8 @@ When a model does not fit entirely in VRAM, the following `options:` control whe
|--------|---------|-------------|
| `backend` | `backend:clip=cpu,vae=cuda0,diffusion=vulkan0` | Runtime (compute) backend assignment per component. Use `cpu` to place a component's compute on the CPU. Component keys include `te` (text encoder / CLIP), `vae`, `diffusion`, `controlnet`. |
| `params_backend` | `params_backend:diffusion=disk,clip=cpu` | Where parameters (weights) are stored. Supports `cpu`, `disk` (mmap weights from disk to save RAM/VRAM), or per-component specs. |
-| `max_vram` | `max_vram:8` or `max_vram:-1` | VRAM budget (in GiB) for graph-cut segmented parameter offload. `0` disables it, `-1` auto-selects (free VRAM minus ~1 GiB). Also accepts per-backend budgets. |
-| `stream_layers` | `stream_layers:true` | Enable residency + prefetch streaming on top of `max_vram` (no effect unless `max_vram` is set). |
+| `max_vram` | `max_vram:8` or `max_vram:-1` | Optional per-device VRAM budget (in GiB) for managed weights and automatic graph-cut execution. `0` uses live free VRAM without an explicit cap; a negative value reserves that many GiB of free VRAM. Also accepts per-backend budgets. |
+| `stream_layers` | `stream_layers:true` | Deprecated compatibility option. Segmented weight streaming is now selected automatically, so this value is ignored. |
| `rpc_servers` | `rpc_servers:localhost:50052,192.168.1.3:50052` | Comma-separated list of `host:port` RPC servers to offload compute to. |
| `pulid_weights_path` | `pulid_weights_path:pulid.safetensors` | Path to PuLID-Flux weights for identity injection. |
diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md
index 8ff355a73..877bd4796 100644
--- a/docs/content/features/text-to-audio.md
+++ b/docs/content/features/text-to-audio.md
@@ -124,6 +124,25 @@ Reference selection follows this order:
When a saved profile is selected, LocalAI supplies both its private WAV and exact transcript for that request. It does not rewrite the model YAML or copy the recording into the model directory.
+### Realtime pipeline default
+
+Set `tts.voice` on a realtime pipeline model to use a saved Voice Library profile as the session default:
+
+```yaml
+name: gpt-realtime
+tts:
+ voice: localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000
+pipeline:
+ vad: silero-vad-ggml
+ transcription: whisper-large-turbo
+ llm: qwen3-4b
+ tts: qwen3-tts-base
+```
+
+LocalAI resolves this profile when the realtime session starts. The selected TTS model must support Voice Library cloning.
+
+This feature does not resolve Voice Library URIs sent later through realtime `session.update`. You can still use `session.update` with ordinary backend voice names or IDs.
+
#### Supported backend and model variants
| Backend | Automatically compatible variants |
diff --git a/docs/content/getting-started/containers.md b/docs/content/getting-started/containers.md
index aa0dfdb50..fed24ebfc 100644
--- a/docs/content/getting-started/containers.md
+++ b/docs/content/getting-started/containers.md
@@ -16,6 +16,10 @@ Before you begin, ensure you have a container engine installed:
- [Install Docker](https://docs.docker.com/get-docker/) (Mac, Windows, Linux)
- [Install Podman](https://podman.io/getting-started/installation) (Linux, macOS, Windows WSL2)
+Podman might not resolve short image names such as `localai/localai:latest`.
+The Podman and Compose examples use the `docker.io/` prefix to identify the
+container registry explicitly.
+
## Quick Start
The fastest way to get started is with the CPU image:
@@ -23,7 +27,7 @@ The fastest way to get started is with the CPU image:
```bash
docker run -p 8080:8080 --name local-ai -ti localai/localai:latest
# Or with Podman:
-podman run -p 8080:8080 --name local-ai -ti localai/localai:latest
+podman run -p 8080:8080 --name local-ai -ti docker.io/localai/localai:latest
```
This will:
@@ -43,7 +47,7 @@ Standard images don't include pre-configured models. Use these if you want to co
```bash
docker run -ti --name local-ai -p 8080:8080 localai/localai:latest
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 localai/localai:latest
+podman run -ti --name local-ai -p 8080:8080 docker.io/localai/localai:latest
```
#### GPU Images
@@ -59,35 +63,35 @@ Choose the image that matches your hardware and installed drivers:
```bash
docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-13
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 --device nvidia.com/gpu=all localai/localai:latest-gpu-nvidia-cuda-13
+podman run -ti --name local-ai -p 8080:8080 --device nvidia.com/gpu=all docker.io/localai/localai:latest-gpu-nvidia-cuda-13
```
**NVIDIA CUDA 12:**
```bash
docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 --device nvidia.com/gpu=all localai/localai:latest-gpu-nvidia-cuda-12
+podman run -ti --name local-ai -p 8080:8080 --device nvidia.com/gpu=all docker.io/localai/localai:latest-gpu-nvidia-cuda-12
```
**AMD GPU (ROCm):**
```bash
docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-gpu-hipblas
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 --device rocm.com/gpu=all localai/localai:latest-gpu-hipblas
+podman run -ti --name local-ai -p 8080:8080 --device rocm.com/gpu=all docker.io/localai/localai:latest-gpu-hipblas
```
**Intel GPU:**
```bash
docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-intel
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 --device gpu.intel.com/all localai/localai:latest-gpu-intel
+podman run -ti --name local-ai -p 8080:8080 --device gpu.intel.com/all docker.io/localai/localai:latest-gpu-intel
```
**Vulkan:**
```bash
docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-vulkan
# Or with Podman:
-podman run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-vulkan
+podman run -ti --name local-ai -p 8080:8080 docker.io/localai/localai:latest-gpu-vulkan
```
**NVIDIA Jetson (L4T ARM64):**
@@ -114,8 +118,8 @@ The CDI approach is recommended for newer versions of the NVIDIA Container Toolk
version: "3.9"
services:
api:
- image: localai/localai:latest-gpu-nvidia-cuda-12
- # For CUDA 13, use: localai/localai:latest-gpu-nvidia-cuda-13
+ image: docker.io/localai/localai:latest-gpu-nvidia-cuda-12
+ # For CUDA 13, use: docker.io/localai/localai:latest-gpu-nvidia-cuda-13
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
# start_period, not timeout, is the knob for a slow first boot: startup
@@ -159,8 +163,8 @@ If you are using an older version of the NVIDIA Container Toolkit (before 1.14),
version: "3.9"
services:
api:
- image: localai/localai:latest-gpu-nvidia-cuda-12
- # For CUDA 13, use: localai/localai:latest-gpu-nvidia-cuda-13
+ image: docker.io/localai/localai:latest-gpu-nvidia-cuda-12
+ # For CUDA 13, use: docker.io/localai/localai:latest-gpu-nvidia-cuda-13
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"]
# start_period, not timeout, is the knob for a slow first boot: startup
@@ -230,7 +234,7 @@ podman run -ti --name local-ai -p 8080:8080 \
-v $PWD/backends:/backends \
-v $PWD/configuration:/configuration \
-v $PWD/data:/data \
- localai/localai:latest
+ docker.io/localai/localai:latest
```
Or use named volumes:
@@ -256,7 +260,7 @@ podman run -ti --name local-ai -p 8080:8080 \
-v localai-backends:/backends \
-v localai-configuration:/configuration \
-v localai-data:/data \
- localai/localai:latest
+ docker.io/localai/localai:latest
```
## Next Steps
@@ -327,9 +331,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master` | `localai/localai:master` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest` | `localai/localai:latest` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}` | `localai/localai:{{< version >}}` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master` | `docker.io/localai/localai:master` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest` | `docker.io/localai/localai:latest` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}` | `docker.io/localai/localai:{{< version >}}` |
{{% /tab %}}
@@ -337,9 +341,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-nvidia-cuda-12` | `localai/localai:master-gpu-nvidia-cuda-12` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-12` | `localai/localai:latest-gpu-nvidia-cuda-12` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-nvidia-cuda-12` | `localai/localai:{{< version >}}-gpu-nvidia-cuda-12` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-nvidia-cuda-12` | `docker.io/localai/localai:master-gpu-nvidia-cuda-12` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-12` | `docker.io/localai/localai:latest-gpu-nvidia-cuda-12` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-nvidia-cuda-12` | `docker.io/localai/localai:{{< version >}}-gpu-nvidia-cuda-12` |
{{% /tab %}}
@@ -347,9 +351,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-nvidia-cuda-13` | `localai/localai:master-gpu-nvidia-cuda-13` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-13` | `localai/localai:latest-gpu-nvidia-cuda-13` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-nvidia-cuda-13` | `localai/localai:{{< version >}}-gpu-nvidia-cuda-13` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-nvidia-cuda-13` | `docker.io/localai/localai:master-gpu-nvidia-cuda-13` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-13` | `docker.io/localai/localai:latest-gpu-nvidia-cuda-13` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-nvidia-cuda-13` | `docker.io/localai/localai:{{< version >}}-gpu-nvidia-cuda-13` |
{{% /tab %}}
@@ -357,9 +361,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-intel` | `localai/localai:master-gpu-intel` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-intel` | `localai/localai:latest-gpu-intel` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-intel` | `localai/localai:{{< version >}}-gpu-intel` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-intel` | `docker.io/localai/localai:master-gpu-intel` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-intel` | `docker.io/localai/localai:latest-gpu-intel` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-intel` | `docker.io/localai/localai:{{< version >}}-gpu-intel` |
{{% /tab %}}
@@ -367,9 +371,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-hipblas` | `localai/localai:master-gpu-hipblas` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-hipblas` | `localai/localai:latest-gpu-hipblas` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-hipblas` | `localai/localai:{{< version >}}-gpu-hipblas` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-hipblas` | `docker.io/localai/localai:master-gpu-hipblas` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-hipblas` | `docker.io/localai/localai:latest-gpu-hipblas` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-hipblas` | `docker.io/localai/localai:{{< version >}}-gpu-hipblas` |
{{% /tab %}}
@@ -377,9 +381,9 @@ The quick-start examples above use the Docker Hub image names. Every image is pu
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-vulkan` | `localai/localai:master-gpu-vulkan` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-vulkan` | `localai/localai:latest-gpu-vulkan` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-vulkan` | `localai/localai:{{< version >}}-gpu-vulkan` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-gpu-vulkan` | `docker.io/localai/localai:master-gpu-vulkan` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-vulkan` | `docker.io/localai/localai:latest-gpu-vulkan` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-gpu-vulkan` | `docker.io/localai/localai:{{< version >}}-gpu-vulkan` |
{{% /tab %}}
@@ -389,9 +393,9 @@ These images are compatible with Nvidia ARM64 devices with CUDA 12, such as the
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64` | `localai/localai:master-nvidia-l4t-arm64` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64` | `localai/localai:latest-nvidia-l4t-arm64` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64` | `localai/localai:{{< version >}}-nvidia-l4t-arm64` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64` | `docker.io/localai/localai:master-nvidia-l4t-arm64` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64` | `docker.io/localai/localai:latest-nvidia-l4t-arm64` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64` | `docker.io/localai/localai:{{< version >}}-nvidia-l4t-arm64` |
{{% /tab %}}
@@ -401,9 +405,9 @@ These images are compatible with Nvidia ARM64 devices with CUDA 13, such as the
| Description | Quay | Docker Hub |
| --- | --- | --- |
-| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-cuda-13` | `localai/localai:master-nvidia-l4t-arm64-cuda-13` |
-| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64-cuda-13` | `localai/localai:latest-nvidia-l4t-arm64-cuda-13` |
-| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64-cuda-13` | `localai/localai:{{< version >}}-nvidia-l4t-arm64-cuda-13` |
+| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-cuda-13` | `docker.io/localai/localai:master-nvidia-l4t-arm64-cuda-13` |
+| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64-cuda-13` | `docker.io/localai/localai:latest-nvidia-l4t-arm64-cuda-13` |
+| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64-cuda-13` | `docker.io/localai/localai:{{< version >}}-nvidia-l4t-arm64-cuda-13` |
{{% /tab %}}
diff --git a/docs/content/integrations.md b/docs/content/integrations.md
index 0bcf69e99..8d8bd4104 100644
--- a/docs/content/integrations.md
+++ b/docs/content/integrations.md
@@ -18,6 +18,15 @@ Feel free to open up a Pull request (by clicking at the "Edit page" below) to ge
- [Helm chart](https://github.com/go-skynet/helm-charts) - Deploy LocalAI on Kubernetes
- [GitHub Actions](https://github.com/marketplace/actions/start-localai) - Use LocalAI in CI/CD workflows
+### Distribution Packages
+
+Community-maintained packages of LocalAI. Package versions and
+availability may lag upstream releases.
+
+- [Homebrew](https://formulae.brew.sh/formula/localai) - `brew install localai`
+- [ALT Sisyphus](https://packages.altlinux.org/en/sisyphus/srpms/localai/) - `localai` package
+- [Gentoo overlay](https://git.ipnmod.org/packages/local-ai-overlay) - `sci-ml/local-ai` and per-backend packages, overlay `local-ai` in Gentoo's repository index
+
### Web UIs
- [localai-admin](https://github.com/Jirubizu/localai-admin)
diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md
index fc2f2a3c9..43ab02dc1 100644
--- a/docs/content/operations/middleware.md
+++ b/docs/content/operations/middleware.md
@@ -45,7 +45,7 @@ routing, `/api/pii/events` for redaction and block actions.
PII redaction is **NER-based and runs request-side (input)**. It is
**off by default**, flipping to **on for any `cloud-proxy` backend**
because that traffic crosses the network to a third-party provider. Pick a
-[default detector](#instance-wide-defaults) so those models are actually
+[default detector](#instance-wide-default-detector) so those models are actually
scanned. Explicit `pii.enabled` in a model's YAML always wins over the
backend default.
diff --git a/docs/content/reference/_index.en.md b/docs/content/reference/_index.en.md
index 654f8b02b..91b090938 100644
--- a/docs/content/reference/_index.en.md
+++ b/docs/content/reference/_index.en.md
@@ -168,8 +168,7 @@ Before using reference documentation, ensure you have:
## Related Sections
- 📖 [Advanced](../advanced/) - Deep dive into configuration and optimization
-- 🔌 [Installation](../installation/) - Setup and deployment
-- ⭐ [Features](../features/) - Feature overview
+- - ⭐ [Features](../features/) - Feature overview
---
diff --git a/docs/content/reference/api-errors.md b/docs/content/reference/api-errors.md
index 4591e0d00..9bd9dea02 100644
--- a/docs/content/reference/api-errors.md
+++ b/docs/content/reference/api-errors.md
@@ -417,4 +417,4 @@ fi
|-------------------------------|------------------------------------------------|
| `LOCALAI_API_KEY` | Comma-separated list of valid API keys |
| `LOCALAI_OPAQUE_ERRORS` | Set to `true` to hide error details (returns empty body with status code only) |
-| `LOCALAI_SUBTLEKEY_COMPARISON`| Use constant-time key comparison for timing-attack resistance |
+| `LOCALAI_SUBTLE_KEY_COMPARISON`| Use constant-time key comparison for timing-attack resistance |
diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md
index fb0872545..d00d8046b 100644
--- a/docs/content/reference/cli-reference.md
+++ b/docs/content/reference/cli-reference.md
@@ -27,9 +27,16 @@ Complete reference for all LocalAI command-line interface (CLI) parameters and e
| `--upload-path` | `TMPDIR/localai-UID/upload` | Path to store uploads from files API. Defaults under the OS temp dir (`$TMPDIR`, falling back to `/tmp`), scoped to the current user's UID. | `$LOCALAI_UPLOAD_PATH`, `$UPLOAD_PATH` |
| `--localai-config-dir` | `BASEPATH/configuration` | Directory for dynamic loading of certain configuration files (currently runtime_settings.json, api_keys.json, and external_backends.json). See [Runtime Settings]({{%relref "features/runtime-settings" %}}) for web-based configuration. | `$LOCALAI_CONFIG_DIR` |
| `--localai-config-dir-poll-interval` | | Time duration to poll the LocalAI Config Dir if your system has broken fsnotify events (example: `1m`) | `$LOCALAI_CONFIG_DIR_POLL_INTERVAL` |
+
| `--models-config-file` | | YAML file containing a list of model backend configs (alias: `--config-file`) | `$LOCALAI_MODELS_CONFIG_FILE`, `$CONFIG_FILE` |
| `--artifact-download-concurrency` | `1` | How many files of a model artifact to download at once. `1` downloads sequentially. Raising it helps artifacts split into many files on a fast link, at the cost of more concurrent load on the models volume. Whole files only — a single file is never split, so resume and per-file checksum verification are unaffected | `$LOCALAI_ARTIFACT_DOWNLOAD_CONCURRENCY` |
+Backend processes receive a private scratch directory through `TMPDIR`, `TMP`,
+and `TEMP`. LocalAI removes that directory when the backend exits and removes
+abandoned directories left by a LocalAI crash before starting another backend.
+Set `$LOCALAI_BACKEND_TEMP_DIR` to choose their base volume. LocalAI always
+appends `localai-UID/backend-runtime`; the default base is `TMPDIR`.
+
## Backend Flags
| Parameter | Default | Description | Environment Variable |
@@ -85,7 +92,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel
| `--max-concurrent-backend-requests` | `1024` | Process-wide ceiling for concurrent backend inference operations. Excess inference receives HTTP 503 with `Retry-After`; UI and administrative endpoints remain available | `$LOCALAI_MAX_CONCURRENT_BACKEND_REQUESTS`, `$MAX_CONCURRENT_BACKEND_REQUESTS` |
| `--cors` | `false` | Enable CORS (Cross-Origin Resource Sharing) | `$LOCALAI_CORS`, `$CORS` |
| `--cors-allow-origins` | | Comma-separated list of allowed CORS origins | `$LOCALAI_CORS_ALLOW_ORIGINS`, `$CORS_ALLOW_ORIGINS` |
-| `--csrf` | `false` | Enable Fiber CSRF middleware | `$LOCALAI_CSRF` |
+| `--disable-csrf` | `false` | Disable CSRF middleware (enabled by default) | `$LOCALAI_DISABLE_CSRF` |
| `--disable-http-compression` | `false` | Disable gzip compression of HTTP responses. Compression is enabled by default; streaming endpoints (streaming chat completions, SSE bridges, WebSocket upgrades) and already-compressed formats are never compressed | `$LOCALAI_DISABLE_HTTP_COMPRESSION` |
| `--http-compression-min-length` | `1024` | Minimum response size in bytes before gzip compression is applied. Smaller responses are sent as-is because the gzip envelope would outweigh the saving | `$LOCALAI_HTTP_COMPRESSION_MIN_LENGTH` |
| `--upload-limit` | `15` | Default upload-limit in MB | `$LOCALAI_UPLOAD_LIMIT`, `$UPLOAD_LIMIT` |
diff --git a/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md b/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md
new file mode 100644
index 000000000..e84d36193
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md
@@ -0,0 +1,158 @@
+# Request-owned ephemeral staging
+
+## Problem
+
+Distributed requests copy transient inputs below
+`/ephemeral//`. The worker currently removes
+these files only when a periodic age sweep considers them stale. A Reachy Mini
+sending camera and sound data about once per second created more than 21,000
+request directories and filled its Mac worker before the six-hour retention
+window elapsed.
+
+Reducing the retention window is insufficient. A time limit bounds residence
+time, but the retained bytes still scale with request rate and input size. A
+quota sweeper would also have to infer whether an old file is still in use.
+Neither rule prevents concurrent uploads from consuming the worker's last free
+space.
+
+## Goals
+
+- Give every ephemeral input an explicit owner and release it when that request
+ finishes, fails, or is cancelled.
+- Keep cleanup transport-independent for HTTP and S3/NATS workers.
+- Reserve capacity before accepting ephemeral bytes so concurrent requests
+ cannot consume configured disk headroom.
+- Reject a request cleanly when its input does not fit; never evict an input
+ that a running request may still be reading.
+- Recover abandoned files after frontend or worker crashes.
+- Never inspect or remove models, data, configuration, or paths outside the
+ worker's ephemeral staging tree.
+
+## Non-goals
+
+- Retaining request inputs as a cache.
+- Evicting persistent model or data files to make an inference request fit.
+- Treating modification timestamps as proof that a request is active.
+
+## Request ownership
+
+The `FileStagingClient` already creates one request ID before staging inputs and
+waits for synchronous and streaming backend calls to finish. It will track each
+ephemeral key before attempting to stage it and defer one request-scoped
+release identified by the request ID. Release runs after the backend call
+returns, including error and cancellation paths, using a short background
+timeout so cancellation of the request does not cancel its cleanup.
+
+Request IDs will use the full UUID rather than the current eight-character
+prefix. The worker enumerates only category directories for that validated
+request ID and removes each entry with exact, symlink-safe deletion.
+
+`FileStager` will expose an idempotent exact-key `ReleaseRemote` operation and
+an optional request-scoped operation. The client uses one fixed-size request
+message for the normal path and retains exact-key calls as a rolling-upgrade
+fallback:
+
+- HTTP sends one authenticated request containing the fixed-size request ID.
+ The worker derives and removes that request's exact files, then prunes empty
+ request and category directories without following symlinks.
+- S3/NATS sends one request-reply containing the request ID so the selected
+ worker evicts the request's local cached files. The frontend then deletes the
+ matching objects from its tracked exact-key list.
+ Either deletion may already have happened and still counts as success.
+
+If staging fails partway through a request, the deferred release still includes
+the planned key, allowing it to remove a partial file when the transport can
+identify one. Cleanup errors are logged and do not replace the inference result.
+
+HTTP and S3 ingress register request operations before any pre-reservation
+work. Before enumerating files, the capacity guard marks the request released
+and waits for registered operations and admitted writes to finish. Later
+operations, reservations, and cache claims for that request are rejected.
+Markers expire after one hour and are capped at 16,384 entries, but a marker is
+never evicted while its registered operation or cleanup scan is active.
+Concurrent operation and cleanup state have the same hard cap. Disk bytes
+remain independently bounded by capacity admission.
+Cleanup waits within its deadline when all cleanup-pin slots are occupied.
+If that deadline expires, existing entries lose active ownership so recovery
+can reclaim them; registered ingress for the request remains closed until it
+exits.
+
+## Capacity admission
+
+A worker-local ephemeral capacity guard is shared by its HTTP and S3/NATS input
+paths. It accounts for both `/ephemeral`, used by HTTP, and
+`/ephemeral`, used by S3 downloads. Before writing an ephemeral object,
+the transport reserves its declared size. HTTP obtains the size from the upload
+metadata; S3/NATS obtains it from object metadata. Reservations are serialized
+in memory, cover both committed ephemeral bytes and concurrent writes, and are
+returned on release or failed transfer.
+
+Admission succeeds only when both conditions remain true after the reservation:
+
+1. Total ephemeral bytes remain below the configured ephemeral staging limit.
+2. The filesystem retains the configured minimum free-space headroom.
+
+The guard rejects the transfer before inference when either condition fails.
+An input with unknown size is written through a bounded accounting writer that
+reserves fixed-size chunks before writing each chunk and stops before crossing
+the limit. The existing maximum-upload-size check remains the per-file ceiling.
+
+The limit and headroom are worker settings. By default, ephemeral data may use
+the smaller of 10 GiB or 10 percent of filesystem capacity, while the worker
+preserves the larger of 1 GiB or 5 percent as free-space headroom. The worker
+logs the effective values at startup. A zero or negative operator value selects
+the default rather than disabling protection. The guard scans the ephemeral
+tree at startup to account for abandoned committed bytes. Filesystem free-space
+checks are repeated at reservation time because other processes may share the
+volume.
+
+## Crash recovery
+
+The existing periodic cleanup remains as a fallback for ownership messages lost
+when a frontend or worker process dies. It uses a one-hour recovery TTL,
+performs one startup sweep, and repeats every 15 minutes. It skips every key
+held by an active reservation, considers the newest modification time in each
+remaining request tree, and does not follow directory symlinks. It removes only
+request directories below the registered `/ephemeral` and
+`/ephemeral` roots.
+
+The recovery window does not control normal storage growth. Request completion
+and capacity reservations do. A recovery deletion updates the capacity guard's
+accounted bytes.
+
+## Error handling and observability
+
+Admission failures report the requested bytes, current ephemeral usage, limit,
+available bytes, and required headroom. Successful release and recovery update
+usage counters. Read, stat, and remove failures include the affected path and
+allow unrelated cleanup to continue. Missing ephemeral files and directories
+are normal for idempotent release.
+
+## Testing
+
+Regression tests will establish the following behavior:
+
+1. Successful, failed, cancelled, and streaming calls issue one request-scoped
+ worker cleanup only after the backend has returned.
+2. Partial staging failures release the planned key without changing the main
+ error returned to the caller.
+3. HTTP and S3/NATS release remove local files; S3/NATS also removes the object.
+4. Release rejects persistent keys and path traversal, does not follow
+ symlinks, and leaves paths outside `ephemeral` untouched.
+5. Concurrent reservations cannot exceed the byte limit or free-space
+ headroom, and failed transfers return their reservations.
+6. Unknown-length writes stop at the capacity boundary.
+7. Startup accounting includes abandoned ephemeral files, and the recovery
+ sweep removes only stale, inactive leftovers and updates accounting.
+
+Focused package tests will run with race detection, followed by the relevant
+repository lint and vet checks.
+
+## Rollout
+
+The change requires a new LocalAI worker and frontend build because both sides
+participate in release. The Mac worker starts by accounting for its existing
+backlog and removing recovery-expired files. The deployment check will verify
+available space, admission and release logs, stable ephemeral usage under
+continuous camera and audio traffic, and successful vision, sound detection,
+and transcription requests.
diff --git a/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md b/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md
new file mode 100644
index 000000000..b4175655f
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md
@@ -0,0 +1,99 @@
+# EXL3 gallery entries
+
+## Goal
+
+Add four gallery entries that expose the EXL3 configurations validated or
+tracked by `vllm.cpp`. Pin each Hugging Face artifact to the revision recorded
+by its source or benchmark evidence.
+
+## Entries
+
+### Qwen3.8 target
+
+Add `qwen3.8-27b-exl3-vllm-cpp` for
+`Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw`. This entry serves the target without a
+draft model.
+
+Use revision `19441ac874c4018295da848e250f23511361cda4`. Configure an 8,192-token
+context, 2,048 cache blocks, eight sequences, and 16,384 batched tokens. Disable
+prefix caching to match the measured serving configuration.
+
+### Qwen3.8 with DFlash2
+
+Add `qwen3.8-27b-dflash2-exl3-vllm-cpp`. This entry stages the Qwen3.8 target
+and `Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw` at revision
+`4f0436269bca761b071f05319e8e04a87cc633f9`.
+
+Configure the `dflash` method with seven speculative tokens. Use the shipped
+paged draft route. Apply the same serving limits as the target-only entry.
+
+Tag this entry with `dflash` because it enables speculative decoding. Declare
+the target-only entry as its variant. LocalAI can then prefer the faster entry
+when the host supports it.
+
+### DeepSeek V4 Flash for Spark
+
+Add `deepseek-v4-flash-spark-exl3-vllm-cpp` for
+`0xSero/deepseek-v4-flash-0731-spark`. Use the current repository revision,
+`ce5ff0f1efb2e184aafc759d281bfae47d3a359c`. State that the `vllm.cpp`
+runtime record used the older revision `22f28d32b9b29b4352eaa380ff8c2c170b2847ab`.
+
+Describe the entry as a Spark and GB10-oriented REAP-K216 checkpoint. State its
+large memory requirement and CUDA requirement. Do not claim a completed speed
+or correctness gate that the source record does not contain.
+
+### DeepSeek V4 Flash 3.0 bpw
+
+Add `deepseek-v4-flash-exl3-3bpw-vllm-cpp` for
+`0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw`. Use the current repository
+revision, `e0bf84ac76a5100e8790c22ad10b70b1e2d06d71`.
+
+Tag and describe this entry as experimental. The model card states that the
+artifact is structurally complete, but end-to-end generation has not passed.
+Keep this entry separate from the Spark entry because the repositories use
+different layouts and have different runtime evidence.
+
+## Artifact staging
+
+Use LocalAI's Hugging Face artifact source for each repository. Stage complete
+model repositories because these safetensors checkpoints need configuration,
+tokenizer, index, and weight files.
+
+Assign the Qwen draft artifact to a companion target. Pass its staged path in
+the `vllm-cpp` speculative configuration. Do not download files through backend
+startup logic.
+
+## User-visible metadata
+
+Use the `vllm-cpp`, `exl3`, `gpu`, and `cuda` tags on all four entries. Add
+architecture, reasoning, tool-calling, and speculative-decoding tags only when
+the configured model supports them.
+
+Descriptions must distinguish measured results from unresolved work. The Qwen
+DFlash2 description can cite the measured configuration and throughput. The
+DeepSeek descriptions must not imply an end-to-end validation that does not
+exist.
+
+## Validation
+
+Run the gallery schema and focused gallery tests. Add a focused test if the
+artifact or variant structure is not already covered.
+
+Validate these properties:
+
+- Every name is unique.
+- Every variant points to an existing entry.
+- Each Hugging Face source has a pinned revision.
+- The DFlash2 entry stages both repositories and passes the draft path.
+- Only the configured DFlash2 entry has the `dflash` tag.
+- YAML parsing and gallery loading succeed.
+
+No model download or GPU benchmark is part of this LocalAI change. The
+`vllm.cpp` evidence supplies the runtime record.
+
+## Out of scope
+
+- Changes to the `vllm-cpp` backend binaries.
+- New EXL3 kernels or model loaders.
+- New benchmark claims.
+- Gallery entries for unselected EXL3 bit widths.
diff --git a/flake.nix b/flake.nix
index 0f632e2d8..ddcdeb26c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -18,6 +18,9 @@
npmRoot = ./core/http/react-ui;
};
npmConfigHook = pkgs.importNpmLock.npmConfigHook;
+ # Avoid EOVERRIDE when importNpmLock rewrites the same-version hono
+ # override to a file: tarball that conflicts with the direct dependency.
+ npmFlags = [ "--legacy-peer-deps" ];
npmBuildScript = "build";
installPhase = ''
diff --git a/formal-verification/README.md b/formal-verification/README.md
index 6bd1da8b0..58bde7a1e 100644
--- a/formal-verification/README.md
+++ b/formal-verification/README.md
@@ -24,11 +24,11 @@ then the implementation is checked against them.
| `fizzbee.sha256` | Pinned checksum(s) of the FizzBee release the gate uses (created on first `install-fizzbee.sh` run). |
The implementations under test live in
-[`core/http/endpoints/openai/respcoord`](../../../core/http/endpoints/openai/respcoord) (M3),
-[`core/http/endpoints/openai/turncoord`](../../../core/http/endpoints/openai/turncoord) (M2),
-[`core/http/endpoints/openai/conncoord`](../../../core/http/endpoints/openai/conncoord) (M1),
-[`core/http/endpoints/openai/compactcoord`](../../../core/http/endpoints/openai/compactcoord) (M4),
-and [`core/http/endpoints/openai/ttscoord`](../../../core/http/endpoints/openai/ttscoord) (M5).
+[`core/http/endpoints/openai/respcoord`](../core/http/endpoints/openai/respcoord) (M3),
+[`core/http/endpoints/openai/turncoord`](../core/http/endpoints/openai/turncoord) (M2),
+[`core/http/endpoints/openai/conncoord`](../core/http/endpoints/openai/conncoord) (M1),
+[`core/http/endpoints/openai/compactcoord`](../core/http/endpoints/openai/compactcoord) (M4),
+and [`core/http/endpoints/openai/ttscoord`](../core/http/endpoints/openai/ttscoord) (M5).
## Running the realtime gate
diff --git a/gallery/index.yaml b/gallery/index.yaml
index 4a0e6afa2..804049539 100644
--- a/gallery/index.yaml
+++ b/gallery/index.yaml
@@ -1,4 +1,789 @@
---
+- name: "qwopus3.8-27b-flash"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF
+ description: |
+ # Qwen3.8-27B
+
+ > [!Note]
+ > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format.
+ >
+ > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc.
+
+ > [!Tip]
+ > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud.
+ > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates.
+
+ Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date.
+
+ ...
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - qwen
+ - qwen3
+ - vision
+ - multimodal
+ - instruction-tuned
+ - reasoning
+ icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/Qwopus3.8-27B-Flash-MTP-Q4_K_M/mmproj-F32.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ model: llama-cpp/models/Qwopus3.8-27B-Flash-MTP-Q4_K_M/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Qwopus3.8-27B-Flash-MTP-Q4_K_M/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf
+ sha256: 2b9a335bd33bb977d3127062061c12cc94f37ceef1850d372317fbf4ee89d9c5
+ uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf
+ - filename: llama-cpp/mmproj/Qwopus3.8-27B-Flash-MTP-Q4_K_M/mmproj-F32.gguf
+ sha256: be3f444ecaedf084fd19399ba18cefcbbe3789bb81f4ad5ed67c91a1a3f48a67
+ uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/mmproj-F32.gguf
+- name: "qwen3.8-27b"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/unsloth/Qwen3.8-27B-GGUF
+ description: |
+ # Qwen3.8-27B
+
+ > [!Note]
+ > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format.
+ >
+ > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc.
+
+ > [!Tip]
+ > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud.
+ > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates.
+
+ Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date.
+
+ ...
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/Qwen3.8-27B-UD-Q4_K_M/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/Qwen3.8-27B-UD-Q4_K_M/Qwen3.8-27B-UD-Q4_K_M.gguf
+ presence_penalty: 1.5
+ repeat_penalty: 1
+ temperature: 0.7
+ top_k: 20
+ top_p: 0.8
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Qwen3.8-27B-UD-Q4_K_M/Qwen3.8-27B-UD-Q4_K_M.gguf
+ sha256: 322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482
+ uri: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-Q4_K_M.gguf
+ - filename: llama-cpp/mmproj/Qwen3.8-27B-UD-Q4_K_M/mmproj-F16.gguf
+ sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e
+ uri: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-F16.gguf
+- name: "glm-5.3-flash"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF
+ description: "# GLM-5.3-Flash\n\n\U0001F44B Join our WeChat or Discord community.\n\n\U0001F4D6 Check out the GLM-5.3-Flash blog and GLM-5 Technical report.\n\n\U0001F4CD Use GLM-5.3-Flash API services on Z.ai API Platform.\n\n## Introduction\n\nWe introduce GLM-5.3-Flash, the first natively multimodal model in the GLM-5 series. With 320B total parameters and just 18B active parameters, it outperforms GLM-5.2 across benchmarks and real-world workloads at one-tenth the price, while approaching Claude Opus 4.8 on coding and agentic benchmarks.\n\nGLM-5.3-Flash starts from a newly trained base model, with its architecture and training recipe redesigned around capability and efficiency. For the first time in the GLM series, we introduce a hybrid architecture combining sparse and linear attention, sharply reducing long-context serving costs while preserving precise long-context capabilities. The model also adopts Manifold-Constrained Hyper-Connections (mHC) to further improve scaling efficiency. Together with our latest 30T-token multimodal pre-training corpus, these changes enable GLM-5.3-Flash to deliver more intelligence with less compute.\n\n## Serve GLM-5.3-Flash Locally\n\n...\n"
+ license: "mit"
+ tags:
+ - llm
+ - gguf
+ icon: https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53.png
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/GLM-5.3-Flash-UD-Q6_K_XL/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0.01
+ model: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf
+ repeat_penalty: 1
+ temperature: 1
+ top_k: -1
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf
+ sha256: 26a4c647133979b9318f2c0a332dac70a3730b374360f590ff125390a556120b
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00002-of-00007.gguf
+ sha256: c65caeffd822aab9810bf1049369f56e7b61c38217fa3c12f3dab2d21c0fd855
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00002-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00003-of-00007.gguf
+ sha256: 491f13fb641ecf27c30107e34a045b8cd00306b43559f3d0273687c47a154559
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00003-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00004-of-00007.gguf
+ sha256: 1db71efa40ba5f9ea017d12d744a8addf4025d3fd256f09e253967327a156270
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00004-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00005-of-00007.gguf
+ sha256: 41acb75b5105d49d89a21c3760d14c458f1e0c2b4e4fff08310c7cb2d5c3fff5
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00005-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00006-of-00007.gguf
+ sha256: 3f669d880bfce522063c677ddddd19f8b066f0d71701d2e123131f239b7fbf6b
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00006-of-00007.gguf
+ - filename: llama-cpp/models/GLM-5.3-Flash-UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00007-of-00007.gguf
+ sha256: 3623f4af825ea01d7cd65706c496ac2c11cb585dfa1a27189320b5790a535b00
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00007-of-00007.gguf
+ - filename: llama-cpp/mmproj/GLM-5.3-Flash-UD-Q6_K_XL/mmproj-F16.gguf
+ sha256: 96ccc182997646ad4405385a1987b1ac1e6adccd2669de43c3ea39692699ed27
+ uri: https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF/resolve/main/mmproj-F16.gguf
+- name: "qwen3.8-27b-uncensored-hauhaucs-aggressive-mtp"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF
+ description: |
+ # Qwen3.8-27B
+
+ > [!Note]
+ > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format.
+ >
+ > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc.
+
+ > [!Tip]
+ > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud.
+ > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates.
+
+ Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date.
+
+ ...
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - vision
+ - multimodal
+ icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P/mmproj-Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+ presence_penalty: 1.5
+ repeat_penalty: 1
+ temperature: 0.7
+ top_k: 20
+ top_p: 0.8
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+ sha256: 4e7735df4d1e2ec721f2551f531b815702a2f89123238c564797eda4b0304bc2
+ uri: https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF/resolve/main/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf
+ - filename: llama-cpp/mmproj/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P/mmproj-Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+ sha256: 5681b690bcb8eb10cd28d62d078cb4e01521a3ea4880a3fc7d54de72de2dd142
+ uri: https://huggingface.co/HauhauCS/Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-MTP-GGUF/resolve/main/mmproj-Qwen3.8-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf
+- name: "qwen3.8-27b-uncensored"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/JonathanColetti/Qwen3.8-27B-Uncensored-GGUF
+ description: |
+ # Qwen3.8-27B
+
+ > [!Note]
+ > This repository contains model weights and configuration files for the post-trained model in the Hugging Face Transformers format.
+ >
+ > These artifacts are compatible with Hugging Face Transformers, vLLM, SGLang, TokenSpeed, etc.
+
+ > [!Tip]
+ > For users seeking managed, scalable inference without infrastructure maintenance, the official Qwen API service is provided by Qwen Cloud.
+ > In particular, **Qwen3.8-27B** will be available as a hosted version with more production features, e.g., 1M context length by default, official built-in tools. For more information, please refer to the Qwen3.8-27B Overview. The service is coming soon. Stay tuned for updates.
+
+ Following the widespread community adoption of the Qwen3.5 and Qwen3.6 series, we are pleased to introduce Qwen3.8, the most capable generation in the Qwen open-model family to date.
+
+ ...
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ icon: https://qianwen-res.oss-accelerate.aliyuncs.com/Qwen3.5/demo/CI_Demo/mathv-1327.jpg
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/Qwen3.8-27B-Uncensored-Q4_K_M/mmproj-Qwen3.8-27B-Uncensored-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/Qwen3.8-27B-Uncensored-Q4_K_M/Qwen3.8-27B-Uncensored-Q4_K_M.gguf
+ presence_penalty: 1.5
+ repeat_penalty: 1
+ temperature: 0.7
+ top_k: 20
+ top_p: 0.8
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Qwen3.8-27B-Uncensored-Q4_K_M/Qwen3.8-27B-Uncensored-Q4_K_M.gguf
+ sha256: 4c5e2db039e9325ac7724c8846c71356a24ad1cdfa28002d73ecb6be645f9675
+ uri: https://huggingface.co/JonathanColetti/Qwen3.8-27B-Uncensored-GGUF/resolve/main/Qwen3.8-27B-Uncensored-Q4_K_M.gguf
+ - filename: llama-cpp/mmproj/Qwen3.8-27B-Uncensored-Q4_K_M/mmproj-Qwen3.8-27B-Uncensored-F16.gguf
+ sha256: 5ac423f8a29059dc24e51bc6a43e9380dcd57a9347f28b62591e0b3f60b7081c
+ uri: https://huggingface.co/JonathanColetti/Qwen3.8-27B-Uncensored-GGUF/resolve/main/mmproj-Qwen3.8-27B-Uncensored-F16.gguf
+- name: "qwen3.8-27b-turbo-fable-cold-fusion-735-882-heretic-uncensored-neo-coder-max-mtp"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/DavidAU/Qwen3.8-27B-TURBO-Fable-Cold-Fusion-735-882-Heretic-Uncensored-NEO-CODER-MAX-MTP-GGUF
+ description: |
+ RELEASE #1 GGUFS [including detailed notes, how to use, benches and much more]:
+ https://huggingface.co/DavidAU/Qwen3.8-27B-TURBO-Fable-Cold-Fusion-735-882-Heretic-Uncensored-NEO-CODER-MAX-MTP-GGUF
+
+ Qwen3.8-27B-TURBO-Fable-Cold-Fusion-735-882-Heretic-Uncensored-NM-DAU (release #1, others pending...)
+
+ ( repo has 10+ other versions (and 3 branches) noted below that EXCEED the performance of all QWEN 27B models, including fine tunes. )
+
+ First, special thanks to Nightmedia for working on the first three stages prior to heretic'ing/post staging and benching everything (3 sections below).
+
+ A number of my finetunes - both released and non-released - were used here as well as some third parties. Full details will be disclosed upon final release as the project shores up.
+
+ THREE example generations [snippets] from STAGE1-PART2, STAGE1b-PART2 and STAGE2-rplus2 at the bottom of the page.
+
+ Release(s) will be GGUFS first (linked here directly) then source code shortly thereafter [now released/open].
+
+ Some additional work and/ spawning of new branches from branch(es) below is still going on.
+
+ NEW: Branch 3 added, see below.
+
+ COMPLETED AND PENDING RELEASES:
+
+ ...
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - reasoning
+ - thinking
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ mmproj: llama-cpp/mmproj/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M/mmproj-F32.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M.gguf
+ presence_penalty: 1.5
+ repeat_penalty: 1
+ temperature: 0.7
+ top_k: 20
+ top_p: 0.8
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M.gguf
+ sha256: bc7a6cf2bcc78d1190aaf04d1ab1c5cb845b6ff23aa0e7d24fe0d2ea6d3a7c7c
+ uri: https://huggingface.co/DavidAU/Qwen3.8-27B-TURBO-Fable-Cold-Fusion-735-882-Heretic-Uncensored-NEO-CODER-MAX-MTP-GGUF/resolve/main/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M.gguf
+ - filename: llama-cpp/mmproj/Qwen3.8-27B-TurboFCFusion-735-882-Here-Uncen-NEO-CODER-MAX-MTP-Q4_K_M/mmproj-F32.gguf
+ sha256: 815ee690ba80c42c0ec589a21792f4e0ed7e15c8d3e47e45a114a4f3951d9963
+ uri: https://huggingface.co/DavidAU/Qwen3.8-27B-TURBO-Fable-Cold-Fusion-735-882-Heretic-Uncensored-NEO-CODER-MAX-MTP-GGUF/resolve/main/mmproj-F32.gguf
+- name: "hy4-preview"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/AngelSlim/Hy4-preview-GGUF
+ description: |
+ # Hy4-preview GGUF
+
+ Three GGUF builds of Hy4-Preview: https://huggingface.co/tencent/Hy4-preview
+
+ **Language / 语言:** English · 中文
+
+ **Neither file runs on stock llama.cpp.** The `hyv4` architecture is not upstream. Apply the
+ patches in `hy4-preview-patch/`
+
+ ## English
+
+ ### 1. What these are
+
+ **`Hy4-preview-Q4_K_M.gguf`** — a conventional Q4_K_M. Most tensors are Q4_K; `ffn_down_exps`
+ gets Q6_K on 37 layers via llama.cpp's own logic. Use this unless you are memory-constrained.
+
+ **`Hy4-preview-UD-IQ1_M.gguf`** - mixed precision with UD-IQ1_M strategy at ~2.44 bpw, roughly **half the size** for the
+ same model. The routed-expert `gate`/`up` projections run at 1.75 bpw (IQ1_M) and 2.0625 bpw (IQ2_XXS).
+
+ **`Hy4-preview-STQ1_0.gguf`** — mixed precision with MIX-STQ1_0 strategy at ~2.38 bpw, roughly **half the size** for the
+ same model. The routed-expert `gate`/`up` projections run at 1.3125 bpw (STQ1_0) on 29 layers and
+ 2.0625 bpw (IQ2_XXS) on the other 48. See section 3.
+
+ ### 2. Running them
+
+ Build a patched llama.cpp
+
+ ```bash
+ git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp
+ git checkout 0cea36222
+
+ ...
+ tags:
+ - llm
+ - gguf
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/Hy4-preview-Q4_K_M/Hy4-preview-Q4_K_M.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Hy4-preview-Q4_K_M/Hy4-preview-Q4_K_M.gguf
+ sha256: 58d0703ef860841dd9b605bfe567af89b62fc7bc367cd57303fc742c6386fffb
+ uri: https://huggingface.co/AngelSlim/Hy4-preview-GGUF/resolve/main/Hy4-preview-Q4_K_M.gguf
+- &apodex-1-1-mini
+ name: "apodex-1.1-mini-q4"
+ variants:
+ - model: apodex-1.1-mini-q4-mtp
+ - model: apodex-1.1-mini-q8
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/apodex/Apodex-1.1-mini
+ - https://huggingface.co/abenzerps/Apodex-1.1-mini-GGUF
+ description: |
+ Apodex-1.1-mini is an Apache-2.0 Qwen3.5 mixture-of-experts model for
+ long-horizon research, data analysis, coding, file work, and tool use. It
+ activates about 3B of its 35.95B parameters per token and supports text and
+ image input with a context window of 262K tokens.
+
+ This default entry uses the recommended Q4_K_M GGUF and F16 vision
+ projector. An MTP-enabled build and a higher-quality Q8_0 model are
+ available as variants.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - qwen
+ - moe
+ - reasoning
+ - thinking
+ - coding
+ - agent
+ - tools
+ - vision
+ - multimodal
+ - long-context
+ last_checked: "2026-08-26"
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q4_K_M.gguf
+ repeat_penalty: 1.05
+ temperature: 1
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q4_K_M.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/Apodex-1.1-mini-Q4_K_M.gguf
+ sha256: 13a580b1ab9350b90085c221cf66e8715a594b2428e5a80a7a956d8e0420be16
+ - filename: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/mmproj-Apodex-1.1-mini-F16.gguf
+ sha256: d0ccf814138020651bcab603d9ce5c080a0ada46f482c116f0aca0fbb84e091e
+- !!merge <<: *apodex-1-1-mini
+ name: "apodex-1.1-mini-q4-mtp"
+ variants: []
+ description: |
+ Apodex-1.1-mini with MTP speculative decoding enabled on the recommended
+ Q4_K_M GGUF. The model carries its native MTP head, so it needs no separate
+ draft model. The F16 vision projector supports multimodal prompts.
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - qwen
+ - moe
+ - reasoning
+ - thinking
+ - coding
+ - agent
+ - tools
+ - vision
+ - multimodal
+ - long-context
+ - mtp
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:3
+ parameters:
+ model: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q4_K_M.gguf
+ repeat_penalty: 1.05
+ temperature: 1
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q4_K_M.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/Apodex-1.1-mini-Q4_K_M.gguf
+ sha256: 13a580b1ab9350b90085c221cf66e8715a594b2428e5a80a7a956d8e0420be16
+ - filename: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/mmproj-Apodex-1.1-mini-F16.gguf
+ sha256: d0ccf814138020651bcab603d9ce5c080a0ada46f482c116f0aca0fbb84e091e
+- !!merge <<: *apodex-1-1-mini
+ name: "apodex-1.1-mini-q8"
+ variants: []
+ description: |
+ Apodex-1.1-mini in the higher-quality Q8_0 GGUF format, with the shared F16
+ vision projector for multimodal prompts.
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q8_0.gguf
+ repeat_penalty: 1.05
+ temperature: 1
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/apodex-1.1-mini/Apodex-1.1-mini-Q8_0.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/Apodex-1.1-mini-Q8_0.gguf
+ sha256: 75340c1561b8e6cb8b6314299a6c4a865ba49f0109e2193dcef48d3478564068
+ - filename: llama-cpp/mmproj/apodex-1.1-mini/mmproj-F16.gguf
+ uri: huggingface://abenzerps/Apodex-1.1-mini-GGUF/mmproj-Apodex-1.1-mini-F16.gguf
+ sha256: d0ccf814138020651bcab603d9ce5c080a0ada46f482c116f0aca0fbb84e091e
+- &glm-5-3-flash-q4
+ name: "glm-5.3-flash-q4"
+ variants:
+ - model: glm-5.3-flash-q8
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/zai-org/GLM-5.3-Flash
+ - https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF
+ description: |
+ GLM-5.3-Flash is Z.ai's natively multimodal 320B-parameter mixture-of-experts
+ model with 18B active parameters. It combines sparse and linear attention
+ for coding, agentic work, tool use, vision, and long-context tasks. This
+ entry uses the UD-Q4_K_XL GGUF quantization and enables the model's MTP
+ speculative-decoding head.
+ license: mit
+ icon: https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/logo.svg
+ tags:
+ - llm
+ - gguf
+ - gpu
+ - cpu
+ - multimodal
+ - vision
+ - reasoning
+ - tool-calling
+ - long-context
+ - moe
+ - mtp
+ - glm
+ - glm-5.3
+ last_checked: "2026-08-30"
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - completion
+ - vision
+ known_input_modalities:
+ - text
+ - image
+ mmproj: llama-cpp/mmproj/glm-5.3-flash/mmproj-BF16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0.01
+ model: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00001-of-00006.gguf
+ repeat_penalty: 1
+ temperature: 1
+ top_k: -1
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00001-of-00006.gguf
+ sha256: 00dceaf3ed08781b1e44513a44ebb19e96248d01ba2a80b17f675a2b6fa9a1ee
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00001-of-00006.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00002-of-00006.gguf
+ sha256: d3ecb6ff3957a99878f9a0352676a0913748a431a9a4a1b1ffa5ade9b8947a74
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00002-of-00006.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00003-of-00006.gguf
+ sha256: 328073c004e7c5395b208247feed26089db247b94dd8fa4ab0f2569d86c1d770
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00003-of-00006.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00004-of-00006.gguf
+ sha256: 6803ab7effa6da4a0b02c9c7865952fdd6c5b2e10e53903348da369e90bd13f9
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00004-of-00006.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00005-of-00006.gguf
+ sha256: 852f3df9dacde3d196796f9fe738468e80c8701555ff0a8c30bc010fb8974dd8
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00005-of-00006.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-UD-Q4_K_XL-00006-of-00006.gguf
+ sha256: c6ba510fafc1e12cc0addbc361a329c0c592ab96980098e7e25e107d8faa983e
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/UD-Q4_K_XL/GLM-5.3-Flash-UD-Q4_K_XL-00006-of-00006.gguf
+ - filename: llama-cpp/mmproj/glm-5.3-flash/mmproj-BF16.gguf
+ sha256: 513c9bfc55898998186543caefc01626fb28e378b92f391018e1c3dd6655b113
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/mmproj-BF16.gguf
+- !!merge <<: *glm-5-3-flash-q4
+ name: "glm-5.3-flash-q8"
+ variants: []
+ description: |
+ GLM-5.3-Flash is Z.ai's natively multimodal 320B-parameter mixture-of-experts
+ model with 18B active parameters. It combines sparse and linear attention
+ for coding, agentic work, tool use, vision, and long-context tasks. This
+ entry uses the higher-quality Q8_0 GGUF quantization and enables the model's
+ MTP speculative-decoding head.
+ overrides:
+ parameters:
+ model: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf
+ files:
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf
+ sha256: 0b21d951c7d14562c479c6a7342c71166d222b21ac2e9c1e32b529a9e017a2df
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00002-of-00008.gguf
+ sha256: 8aabbbc0a92c551894f62a0737b433c9fad60a94baaa61e669f96329bb1d88a6
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00002-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00003-of-00008.gguf
+ sha256: 7b50941b404fed3413be4e0ca52af43e79113e4b3e2c940d1f4cdf74e95ebb39
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00003-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00004-of-00008.gguf
+ sha256: ff27fb4536d524cae687c4fb88dfa1b43a8ee0b4256e55304f95dc1aa38e8b81
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00004-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00005-of-00008.gguf
+ sha256: c088636854c5da312e44a7be4195cd765dd9eb6339d7b014d1da4192851a9b5b
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00005-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00006-of-00008.gguf
+ sha256: 4bece55cb9e98720f3ef2e1e9683983aeced77434dbcd54584c0b7c80a2e95cb
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00006-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00007-of-00008.gguf
+ sha256: a9f792204b0b58155e6092f9a4160a0659fcdc98a1c126e563ecc3cd02ad68ad
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00007-of-00008.gguf
+ - filename: llama-cpp/models/glm-5.3-flash/GLM-5.3-Flash-Q8_0-00008-of-00008.gguf
+ sha256: a6cb6887c0047653f08c0ab6e564972451d9511322202970f7052e36052aefd6
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/Q8_0/GLM-5.3-Flash-Q8_0-00008-of-00008.gguf
+ - filename: llama-cpp/mmproj/glm-5.3-flash/mmproj-BF16.gguf
+ sha256: 513c9bfc55898998186543caefc01626fb28e378b92f391018e1c3dd6655b113
+ uri: huggingface://unsloth/GLM-5.3-Flash-GGUF/mmproj-BF16.gguf
+- name: "nl2sh-1.5b-q4"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct
+ - https://huggingface.co/ThorOdinson246/nl2sh-1.5b-Q4_K_M
+ description: |
+ nl2sh-1.5b is a 1.5B Qwen2.5-Coder fine-tune that converts plain-English
+ requests into single POSIX or Bash commands. This Q4_K_M GGUF is 941 MB
+ and is designed for fast CPU inference.
+
+ Use the system prompt from the model card and review every generated
+ command before execution. The model can produce destructive commands and
+ cannot inspect the local filesystem.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - coding
+ - shell
+ - command-generation
+ last_checked: "2026-08-15"
+ overrides:
+ backend: llama-cpp
+ context_size: 32768
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: nl2sh-1.5b-Q4_K_M.gguf
+ temperature: 0
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: nl2sh-1.5b-Q4_K_M.gguf
+ uri: huggingface://ThorOdinson246/nl2sh-1.5b-Q4_K_M/nl2sh-1.5b-Q4_K_M.gguf
+ sha256: 6f8a17a11129a31074c944f4c2602453fafd9de43bdaeb1630a8f511ec820f71
+- &s1-mini
+ name: "s1-mini-q4"
+ variants:
+ - model: s1-mini-f16
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/superwhisper/s1-mini
+ - https://huggingface.co/superwhisper/s1-mini-GGUF
+ description: |
+ S1-mini by Superwhisper is a 0.6B English text normalizer for raw speech
+ transcripts. It removes fillers and false starts, restores punctuation and
+ capitalization, and formats spoken numbers, dates, currency, and email
+ addresses as written text.
+
+ This default entry uses the publisher's 462 MB Q4_K_M GGUF and greedy
+ decoding. A higher-fidelity F16 model is available as a variant. Prefix the
+ transcript with the styling, structure, and context control line documented
+ on the model page.
+ license: "s1-mini-license"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - qwen3
+ - english
+ - text-normalization
+ - asr-post-processing
+ icon: https://huggingface.co/superwhisper/s1-mini/resolve/main/banner.jpg
+ last_checked: "2026-08-20"
+ overrides:
+ backend: llama-cpp
+ chat_template_kwargs:
+ enable_thinking: false
+ context_size: 4096
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/s1-mini/s1-mini-q4_k_m.gguf
+ temperature: 0
+ system_prompt: >-
+ You are a text normalizer for speech-to-text transcripts. The input begins with a control line specifying the styling, structure, and context settings; clean the transcript to match those settings and output only the cleaned text.
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/s1-mini/s1-mini-q4_k_m.gguf
+ uri: huggingface://superwhisper/s1-mini-GGUF/s1-mini-q4_k_m.gguf
+ sha256: 3b41ebe2502cbd03e811d5d16b022f5ab551eda58d62597d152f89535003c634
+- !!merge <<: *s1-mini
+ name: "s1-mini-f16"
+ variants: []
+ description: |
+ S1-mini by Superwhisper in the publisher's 1.4 GB F16 GGUF format. This
+ variant preserves full model fidelity for hosts with enough memory.
+ overrides:
+ backend: llama-cpp
+ chat_template_kwargs:
+ enable_thinking: false
+ context_size: 4096
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/s1-mini/s1-mini-f16.gguf
+ temperature: 0
+ system_prompt: >-
+ You are a text normalizer for speech-to-text transcripts. The input begins with a control line specifying the styling, structure, and context settings; clean the transcript to match those settings and output only the cleaned text.
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/s1-mini/s1-mini-f16.gguf
+ uri: huggingface://superwhisper/s1-mini-GGUF/s1-mini-f16.gguf
+ sha256: 0370da4f1bae19e3150bcafa33c5d396c15f97bf25519540a3e013db5cc00af4
- name: "glm-5.3"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
urls:
@@ -104,6 +889,110 @@
- filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf
sha256: 79af2211278ac07dfe4c789751d7de7d7caf0a4516486f8a9b6eedb1f3fb6e9b
uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf
+- name: "supra2-100m-instruct"
+ url: "github:mudler/LocalAI/gallery/qwen3.yaml@master"
+ urls:
+ - https://huggingface.co/SupraLabs/Supra2-100M-Instruct
+ description: |
+ Supra2-100M-Instruct is a compact English chat model trained from scratch by
+ SupraLabs on the Qwen3 architecture. It has 100 million parameters, a
+ 2,048-token context window, and is intended for lightweight experiments and
+ constrained edge deployments. This entry uses the publisher's official F16
+ GGUF build.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - chat
+ - qwen3
+ - edge
+ - english
+ last_checked: "2026-08-11"
+ overrides:
+ parameters:
+ context_size: 2048
+ model: Supra2-100M-SFT-F16.gguf
+ files:
+ - filename: Supra2-100M-SFT-F16.gguf
+ uri: huggingface://SupraLabs/Supra2-100M-Instruct/Supra2-100M-SFT-F16.gguf
+ sha256: f88228c3dcc13b5ee2333cbb6fc2416de908e5b8adabd39bf9ee4378f28451eb
+- &llm-jp-4-33b-thinking
+ name: "llm-jp-4-33b-thinking-q4"
+ variants:
+ - model: llm-jp-4-33b-thinking-bf16
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/llm-jp/llm-jp-4-33b-thinking
+ - https://huggingface.co/llm-jp/llm-jp-4-33b-thinking-gguf
+ description: |
+ LLM-jp-4-33B-thinking is an Apache-2.0 Japanese and English reasoning
+ model from Japan's National Institute of Informatics. Its dense Llama
+ architecture has 33 billion parameters and a 65K-token context window.
+ The model was aligned with supervised fine-tuning and DPO for multi-turn
+ conversation and instruction following.
+
+ This default entry uses the 20.2 GB Q4_K_M GGUF. The official 66.4 GB
+ BF16 weights are available as a higher-fidelity variant.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - multilingual
+ - japanese
+ - reasoning
+ - thinking
+ - tools
+ - long-context
+ last_checked: "2026-08-23"
+ overrides:
+ backend: llama-cpp
+ context_size: 65536
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-Q4_K_M.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-Q4_K_M.gguf
+ uri: huggingface://llm-jp/llm-jp-4-33b-thinking-gguf/llm-jp-4-33b-thinking-Q4_K_M.gguf
+ sha256: 9e48892c0d5ec256d05fc3258c9e50738852c39636fbb9fe1f4b1cde2f5e7520
+- !!merge <<: *llm-jp-4-33b-thinking
+ name: "llm-jp-4-33b-thinking-bf16"
+ variants: []
+ description: |
+ LLM-jp-4-33B-thinking in the official 66.4 GB BF16 GGUF format. This
+ variant preserves the original model precision for hosts with enough
+ memory.
+ overrides:
+ backend: llama-cpp
+ context_size: 65536
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-BF16.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/llm-jp-4-33b-thinking/llm-jp-4-33b-thinking-BF16.gguf
+ uri: huggingface://llm-jp/llm-jp-4-33b-thinking-gguf/llm-jp-4-33b-thinking-BF16.gguf
+ sha256: 7bb8465702b5c4a5d94e03e62921d5917181a2eda56da31edfdcd47fab2e9964
- &qwen3-8-flash-next
name: "qwen3.8-flash-next-q4"
variants:
@@ -309,6 +1198,35 @@
- embeddings
parameters:
model: tencent/WeMM-Embedding-9B
+- name: "dfm-mimir:vllm"
+ url: github:mudler/LocalAI/gallery/vllm.yaml@master
+ urls:
+ - https://huggingface.co/danish-foundation-models/DFM-Mimir
+ description: |
+ DFM Mimir is an Apache-2.0, instruction-tuned HRM-Text model from Danish
+ Foundation Models. It has about 1 billion parameters and a 4,096-token
+ context window. The model focuses on Danish and English chat, reasoning,
+ mathematics, and code generation, and uses only permissible post-training
+ data. This entry serves the official BF16 safetensors checkpoint with vLLM.
+ license: apache-2.0
+ tags:
+ - llm
+ - chat
+ - reasoning
+ - coding
+ - instruction-tuned
+ - danish
+ - english
+ - hrm-text
+ - safetensors
+ - vllm
+ - gpu
+ last_checked: "2026-08-25"
+ overrides:
+ context_size: 4096
+ parameters:
+ max_tokens: 4096
+ model: danish-foundation-models/DFM-Mimir
- &granite-4-2-3b
name: "granite-4.2-3b-q4"
variants:
@@ -353,7 +1271,7 @@
files:
- filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q4_K_M.gguf
uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q4_K_M.gguf
- sha256: 20e436143017578687f7f848225cc6c6038126c84149192229c7dff6e4e0f427
+ sha256: e0406663965846ae22a403456eb826ccce5f450840491f71952f18a7cb78e7d5
- !!merge <<: *granite-4-2-3b
name: "granite-4.2-3b-q8"
variants: []
@@ -378,7 +1296,7 @@
files:
- filename: llama-cpp/models/granite-4.2-3b/granite-4.2-3b-Q8_0.gguf
uri: huggingface://ibm-granite/granite-4.2-3b-GGUF/granite-4.2-3b-Q8_0.gguf
- sha256: 9e97320b131445ab8d9098cafb48001e9925d879e71486a8af4db4c803c55394
+ sha256: fbe986738041418e26de9e123ba740cb654931f85bf572a71bd01f9e6b85e53d
- &granite-4-2-8b
name: "granite-4.2-8b-q4"
variants:
@@ -520,6 +1438,109 @@
- filename: llama-cpp/models/granite-4.2-30b/granite-4.2-30b-Q8_0.gguf
uri: huggingface://ibm-granite/granite-4.2-30b-GGUF/granite-4.2-30b-Q8_0.gguf
sha256: 005b0933353e9ba219b26e2667705bdb8dbc74eb50e4a4e6cb70fca108710f81
+- &dirk-qwen3-8-27b
+ name: "dirk-qwen3.8-27b-q4"
+ variants:
+ - model: dirk-qwen3.8-27b-q8
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/Qwen/Qwen3.8-27B
+ - https://huggingface.co/peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF
+ description: |
+ Dirk is a Qwen3.8 27B vision-language model with a concise chat template
+ for agentic coding, reasoning, tool use, and general knowledge tasks. It
+ preserves the model's MTP head for speculative decoding and supports a
+ 262K-token context window.
+
+ This default entry uses the Q4_K_XL GGUF and F16 vision projector. A
+ higher-quality Q8_K_XL build is available as a variant.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - qwen
+ - reasoning
+ - thinking
+ - coding
+ - agent
+ - tools
+ - vision
+ - multimodal
+ - long-context
+ - mtp
+ last_checked: "2026-08-31"
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf
+ repeat_penalty: 1
+ temperature: 0.6
+ top_k: 20
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf
+ uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/Dirk-Qwen3.8-27B-UD-Q4_K_XL.gguf
+ sha256: d1ad2472a147caa1111bae5ec710331dc50692d62ebbdb3fbc54d421c4e209bc
+ - filename: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf
+ uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/mmproj-F16.gguf
+ sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e
+- !!merge <<: *dirk-qwen3-8-27b
+ name: "dirk-qwen3.8-27b-q8"
+ variants: []
+ description: |
+ Dirk in the higher-quality Q8_K_XL GGUF format, with MTP speculative
+ decoding and the shared F16 vision projector for multimodal prompts.
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf
+ repeat_penalty: 1
+ temperature: 0.6
+ top_k: 20
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/dirk-qwen3.8-27b/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf
+ uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/Dirk-Qwen3.8-27B-UD-Q8_K_XL.gguf
+ sha256: be2f08a260021fb471f91f2c5a52aa4afe143688e8b412beb5a36cad2d47c4cd
+ - filename: llama-cpp/mmproj/dirk-qwen3.8-27b/mmproj-F16.gguf
+ uri: huggingface://peculiar-ragdoll/Dirk-Qwen3.8-27B-GGUF/mmproj-F16.gguf
+ sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e
- name: "qwen3.8-27b-dflash2"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
urls:
@@ -1718,6 +2739,7 @@
name: "qwen3.8-27b-q4"
variants:
- model: qwen3.8-27b-q4-mtp
+ - model: qwen3.8-27b-nvfp4-mtp
- model: qwen3.8-27b-ridge
- model: qwen3.8-27b-gsq-rco-iq2-xs
- model: qwen3.8-27b-gsq-rco-iq2-s
@@ -1815,7 +2837,7 @@
known_usecases:
- chat
- vision
- mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf
+ mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf
options:
- use_jinja:true
- spec_type:draft-mtp
@@ -1841,6 +2863,68 @@
- filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf
uri: huggingface://ggml-org/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf
sha256: 2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb
+- !!merge <<: *qwen3-8-27b
+ name: "qwen3.8-27b-nvfp4-mtp"
+ variants: []
+ urls:
+ - https://huggingface.co/Qwen/Qwen3.8-27B
+ - https://huggingface.co/esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF
+ description: |
+ Qwen3.8-27B in a compact NVFP4 GGUF format with its MTP draft head
+ embedded in the model file. This entry uses the medium tier, which keeps
+ the NVFP4 backbone while using higher-precision output and embedding
+ tensors. MTP speculative decoding proposes multiple tokens for the target
+ model to verify.
+ tags:
+ - llm
+ - gguf
+ - gpu
+ - qwen
+ - reasoning
+ - thinking
+ - coding
+ - agent
+ - tools
+ - vision
+ - multimodal
+ - long-context
+ - nvfp4
+ - mtp
+ - speculative
+ last_checked: "2026-08-19"
+ overrides:
+ backend: llama-cpp
+ context_size: 262144
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - vision
+ mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf
+ options:
+ - use_jinja:true
+ - spec_type:draft-mtp
+ - spec_n_max:6
+ - spec_p_min:0.75
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf
+ presence_penalty: 0
+ repeat_penalty: 1
+ temperature: 1
+ top_k: 20
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf
+ uri: huggingface://esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF/Qwen3.8-27B-NVFP4-MTP-MEDIUM.gguf
+ sha256: f0b4c538c75037f026bde3b650f0ca639d382c128a4572769dce1183db86253a
+ - filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf
+ uri: huggingface://esatapedico/Qwen3.8-27B-NVFP4-MTP-GGUF/mmproj-BF16.gguf
+ sha256: 83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53
- !!merge <<: *qwen3-8-27b
name: "qwen3.8-27b-q8"
variants: []
@@ -3070,6 +4154,85 @@
- filename: llama-cpp/models/qwen3.8-4b-distill/Qwen3.8-4B-Q8_0.gguf
uri: huggingface://empero-ai/Qwen3.8-4B-Distill-GGUF/Qwen3.8-4B-Q8_0.gguf
sha256: 770b780d6754a4954d1caf395c9239eaeb394f15c7a7ea34039883377c93c9c3
+- &qwen3-8-9b-distill
+ name: "qwen3.8-9b-distill-q4"
+ variants:
+ - model: qwen3.8-9b-distill-q8
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/empero-ai/Qwen3.8-9B-Distill
+ - https://huggingface.co/empero-ai/Qwen3.8-9B-Distill-GGUF
+ description: |
+ Qwen3.8 9B Distill is an Apache-2.0, text-only Qwen3.5 9B fine-tune
+ distilled from Qwen3.8 2.4T A95B reasoning traces. It targets mathematics,
+ coding, instruction following, and function calling with a 262K native
+ context window. This entry uses the balanced Q4_K_M GGUF quantization; the
+ Q8_0 variant offers higher fidelity.
+ license: apache-2.0
+ icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - qwen3.5
+ - reasoning
+ - coding
+ - tool-use
+ last_checked: "2026-08-20"
+ overrides:
+ backend: llama-cpp
+ context_size: 32768
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ - reasoning_format:deepseek
+ parameters:
+ model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q4_K_M.gguf
+ temperature: 0.6
+ top_k: 20
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q4_K_M.gguf
+ uri: huggingface://empero-ai/Qwen3.8-9B-Distill-GGUF/Qwen3.8-9B-Q4_K_M.gguf
+ sha256: df13d66021cef676f82be74053220fd75af6bf2a6a7fb77f5222ab9e50744a7a
+- !!merge <<: *qwen3-8-9b-distill
+ name: "qwen3.8-9b-distill-q8"
+ variants: []
+ description: |
+ Qwen3.8 9B Distill in the higher-fidelity Q8_0 GGUF format. This text-only
+ Qwen3.5 9B fine-tune targets reasoning, coding, instruction following, and
+ function calling with a 262K native context window.
+ overrides:
+ backend: llama-cpp
+ context_size: 32768
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ - reasoning_format:deepseek
+ parameters:
+ model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q8_0.gguf
+ temperature: 0.6
+ top_k: 20
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q8_0.gguf
+ uri: huggingface://empero-ai/Qwen3.8-9B-Distill-GGUF/Qwen3.8-9B-Q8_0.gguf
+ sha256: 79ca5d342a07922f2bbf38c8d892a79a3c8620c65feaf4b1c66b7830ae724db8
- name: "btl-4-compact"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
urls:
@@ -3119,6 +4282,80 @@
- filename: llama-cpp/models/BTL-4-Compact/BTL-4-IQ2_XXS.gguf
uri: huggingface://badtheorylabs/BTL-4-Compact/BTL-4-IQ2_XXS.gguf
sha256: 6b7c298cf909fc04428ecf360a29dcc578188b1c90aa6ed435159f5a0d351496
+- &mxbai-embed-large-v1
+ name: "mxbai-embed-large-v1-q4"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ variants:
+ - model: mxbai-embed-large-v1-q8
+ - model: mxbai-embed-large-v1-f16
+ license: apache-2.0
+ urls:
+ - https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1
+ - https://huggingface.co/ChristianAzinn/mxbai-embed-large-v1-gguf
+ description: |
+ Mixedbread's mxbai-embed-large-v1 is a 335M-parameter English BERT
+ embedding model for retrieval, semantic search, and RAG. It produces
+ 1,024-dimensional embeddings and supports sequences up to 512 tokens.
+ Prefix retrieval queries with `Represent this sentence for searching
+ relevant passages: `. This entry uses the balanced Q4_K_M GGUF.
+ tags:
+ - embeddings
+ - retrieval
+ - rag
+ - gguf
+ - cpu
+ - gpu
+ - english
+ last_checked: "2026-08-27"
+ overrides:
+ backend: llama-cpp
+ embeddings: true
+ known_usecases:
+ - embeddings
+ parameters:
+ model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q4_K_M.gguf
+ files:
+ - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q4_K_M.gguf
+ uri: huggingface://ChristianAzinn/mxbai-embed-large-v1-gguf/mxbai-embed-large-v1.Q4_K_M.gguf
+ sha256: 3869d462819e3f6cd2c1b0f8d6817e95cc1ed31fc09388432669209d1c6f1b65
+- !!merge <<: *mxbai-embed-large-v1
+ name: "mxbai-embed-large-v1-q8"
+ variants: []
+ description: |
+ Mixedbread's mxbai-embed-large-v1 in the higher-fidelity Q8_0 GGUF
+ format. This 335M-parameter English BERT model produces
+ 1,024-dimensional embeddings for retrieval, semantic search, and RAG.
+ overrides:
+ backend: llama-cpp
+ embeddings: true
+ known_usecases:
+ - embeddings
+ parameters:
+ model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q8_0.gguf
+ files:
+ - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1.Q8_0.gguf
+ uri: huggingface://ChristianAzinn/mxbai-embed-large-v1-gguf/mxbai-embed-large-v1.Q8_0.gguf
+ sha256: bcdebca12aa16c0e51d166d97e4776efd46f905952b0c9acb968976eba2619f3
+- !!merge <<: *mxbai-embed-large-v1
+ name: "mxbai-embed-large-v1-f16"
+ variants: []
+ urls:
+ - https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1
+ description: |
+ Mixedbread's mxbai-embed-large-v1 in the official full-precision F16
+ GGUF format. This 335M-parameter English BERT model produces
+ 1,024-dimensional embeddings for retrieval, semantic search, and RAG.
+ overrides:
+ backend: llama-cpp
+ embeddings: true
+ known_usecases:
+ - embeddings
+ parameters:
+ model: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1-f16.gguf
+ files:
+ - filename: llama-cpp/models/mxbai-embed-large-v1/mxbai-embed-large-v1-f16.gguf
+ uri: huggingface://mixedbread-ai/mxbai-embed-large-v1/gguf/mxbai-embed-large-v1-f16.gguf
+ sha256: 819c2adf5ce6df2b6bd2ae4ca90d2a69f060afeb438d0c171db57daa02e39c3d
- &nemotron-3-embed-1b
name: "nemotron-3-embed-1b-q4"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
@@ -4115,7 +5352,7 @@
uri: https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF/resolve/main/Hermes3.6-35B-A3B-Uncensored-Genesis-V6-Q8_0.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V6-Q8_0/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: https://huggingface.co/LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V6-GGUF/resolve/main/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- &qwen3-6-35b-a3b-genesis-hermes-v7
name: "qwen3.6-35b-a3b-genesis-hermes-v7"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
@@ -4177,7 +5414,7 @@
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-APEX.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7
name: "qwen3.6-35b-a3b-genesis-hermes-v7-apex-compact"
variants: []
@@ -4213,7 +5450,7 @@
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-APEX-Compact.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7
name: "qwen3.6-35b-a3b-genesis-hermes-v7-mtp-apex"
variants: []
@@ -4261,7 +5498,7 @@
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-MTP-APEX.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7
name: "qwen3.6-35b-a3b-genesis-hermes-v7-mtp-apex-compact"
variants: []
@@ -4309,7 +5546,7 @@
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-MTP-APEX-Compact.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- !!merge <<: *qwen3-6-35b-a3b-genesis-hermes-v7
name: "qwen3.6-35b-a3b-genesis-hermes-v7-q8-k-p"
variants: []
@@ -4344,7 +5581,7 @@
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/Hermes3.6-35B-A3B-Uncensored-Genesis-V7-Q8_K_P.gguf
- filename: llama-cpp/mmproj/Hermes3.6-35B-A3B-Uncensored-Genesis-V7/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
uri: huggingface://LuffyTheFox/Qwen3.6-35B-A3B-Uncensored-Genesis-Hermes-V7-GGUF/mmproj-Hermes3.6-35B-A3B-Uncensored-Genesis-F16.gguf
- sha256: ce49c492921cbf22991a7e3f927fa489ed7f87e1e426f542ce96b317c21072b4
+ sha256: f7197461d8581cd9be42384d4afe3f851d0844f6d72846d5c97e3dcc931b3cbc
- &kat-coder-v2-5-dev
name: "kat-coder-v2.5-dev"
variants:
@@ -6257,6 +7494,8 @@
variants:
- model: laguna-s-2.1-q8
- model: laguna-s-2.1-dflash
+ - model: laguna-s-2.1-apex-i-quality
+ - model: laguna-s-2.1-apex-i-compact
description: |
Laguna S 2.1 is Poolside's 118B-parameter, 8B-active Mixture-of-Experts model for agentic software engineering. It supports tool use and a native one-million-token context window; the official GGUF recommends 256K context for best output quality. This default entry uses the current 96 GB Q4_K_M artifact, with imatrix-quantized routed experts and a Q8_0 signal path.
@@ -6280,6 +7519,70 @@
- filename: llama-cpp/models/Laguna-S-2.1-GGUF/laguna-s-2.1-Q4_K_M.gguf
sha256: a8b55c75714ea73fd90ec85de5defdc0b8d88ca0ad2108343cdd8fc22f7583e4
uri: https://huggingface.co/poolside/Laguna-S-2.1-GGUF/resolve/main/laguna-s-2.1-Q4_K_M.gguf
+- !!merge <<: *laguna-s-2-1-q8
+ name: "laguna-s-2.1-apex-i-quality"
+ variants: []
+ urls:
+ - https://huggingface.co/poolside/Laguna-S-2.1
+ - https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF
+ description: |
+ Laguna S 2.1 in the 73.9 GB APEX-I Quality format. This community build
+ uses an importance matrix and mixed precision to preserve the always-active
+ signal path while reducing the memory required by the routed experts.
+
+ License: OpenMDW 1.1.
+ last_checked: "2026-08-26"
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - completion
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf
+ sha256: 772886614b2f11ba62ea2c2ebb5d02804e1ab829329b50f644e83e174decfaf2
+ uri: huggingface://Myric/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-quality-v2.gguf
+- !!merge <<: *laguna-s-2-1-q8
+ name: "laguna-s-2.1-apex-i-compact"
+ variants: []
+ urls:
+ - https://huggingface.co/poolside/Laguna-S-2.1
+ - https://huggingface.co/Myric/Laguna-S-2.1-APEX-GGUF
+ description: |
+ Laguna S 2.1 in the smaller 54.4 GB APEX-I Compact format. This community
+ build uses an importance matrix and mixed precision to reduce memory use
+ while retaining higher precision for the always-active signal path.
+
+ License: OpenMDW 1.1.
+ last_checked: "2026-08-26"
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ - completion
+ options:
+ - use_jinja:true
+ parameters:
+ model: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf
+ sha256: b5964de8c5de37c9d4fcbd6894a68a95a81a2c404b12897d9581fc0d934134de
+ uri: huggingface://Myric/Laguna-S-2.1-APEX-GGUF/Laguna-S-2.1-APEX-i-compact-v2.gguf
- !!merge <<: *laguna-s-2-1-q8
name: "laguna-s-2.1-dflash"
description: |
@@ -8182,6 +9485,51 @@
- filename: llama-cpp/mmproj/gemma-4-12B-it-qat-q4_0-gguf/mmproj-gemma-4-12b-it-qat-q4_0.gguf
uri: https://huggingface.co/google/gemma-4-12B-it-qat-q4_0-gguf/resolve/main/mmproj-gemma-4-12b-it-qat-q4_0.gguf
sha256: cb018338a7538a9814d994bfe54644c71eb7ed54e31eae2f721e45fd3c260da7
+- name: "security-slm-gemma-4-e2b-it-q4"
+ url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
+ urls:
+ - https://huggingface.co/entrick/Security-SLM-Gemma-4-E2B-it-GGUF
+ - https://huggingface.co/unsloth/gemma-4-E2B-it-unsloth-bnb-4bit
+ description: |
+ Security-SLM is a compact Gemma 4 E2B fine-tune for authorized red-team,
+ blue-team, security operations, and AI security work. It is designed for
+ private and air-gapped deployments where prompts can contain sensitive
+ incident data, policies, or source code.
+
+ This entry uses the text-only 3.43 GB Q4_K_M GGUF release.
+ license: "apache-2.0"
+ tags:
+ - llm
+ - gguf
+ - cpu
+ - gpu
+ - gemma
+ - cybersecurity
+ - tools
+ last_checked: "2026-08-31"
+ overrides:
+ backend: llama-cpp
+ function:
+ automatic_tool_parsing_fallback: true
+ grammar:
+ disable: true
+ known_usecases:
+ - chat
+ options:
+ - use_jinja:true
+ parameters:
+ min_p: 0
+ model: llama-cpp/models/security-slm-gemma-4-e2b-it/security-gemma-4-e2b-it.Q4_K_M.gguf
+ repeat_penalty: 1
+ temperature: 1
+ top_k: 64
+ top_p: 0.95
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: llama-cpp/models/security-slm-gemma-4-e2b-it/security-gemma-4-e2b-it.Q4_K_M.gguf
+ uri: huggingface://entrick/Security-SLM-Gemma-4-E2B-it-GGUF/security-gemma-4-e2b-it.Q4_K_M.gguf
+ sha256: 9046894db49c38390088d151dd73397ce1a948480862b3fb8fcbb1b9e977ee5e
- name: "gemma-4-e2b-it-qat-q4_0"
url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
urls:
@@ -12762,8 +14110,8 @@
model: kokoro-int8-multi-lang-v1_0/model.int8.onnx
files:
- filename: kokoro-int8-multi-lang-v1_0.tar.bz2
- sha256: 75654a84864be26f345f020f4070c2c019e96dd1b7f9bf6e2ffd59efac6aa5a3
uri: https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-int8-multi-lang-v1_0.tar.bz2
+ sha256: 4c3052abaa60943a341f193888cf6abd68787dae6ab8ae5c925a706caa247e4e
- name: supertonic-3
url: github:mudler/LocalAI/gallery/supertonic.yaml@master
urls:
@@ -12865,6 +14213,36 @@
source:
type: huggingface
repo: openbmb/VoxCPM1.5
+- name: voxcpm2
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/openbmb/VoxCPM2
+ description: |
+ VoxCPM2 is a 2B-parameter text-to-speech model supporting 30 languages and 48 kHz output, with voice design and controllable voice cloning.
+ license: apache-2.0
+ icon: https://cdn-avatars.huggingface.co/v1/production/uploads/1670387859384-633fe7784b362488336bbfad.png
+ tags:
+ - tts
+ - text-to-speech
+ - voice-cloning
+ - voice-design
+ - cpu
+ - gpu
+ last_checked: "2026-08-22"
+ overrides:
+ backend: voxcpm
+ known_usecases:
+ - tts
+ tts:
+ voice_cloning: true
+ parameters:
+ model: openbmb/VoxCPM2
+ artifacts:
+ - name: model
+ target: model
+ source:
+ type: huggingface
+ repo: openbmb/VoxCPM2
- name: neutts-air
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:
@@ -13085,6 +14463,202 @@
- vae/**
parameters:
model: meituan-longcat/LongCat-Video-Avatar-1.5
+- name: qwen3.8-27b-exl3-vllm-cpp
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ - https://github.com/mudler/vllm.cpp
+ description: |
+ Qwen3.8-27B EXL3 3.5bpw served by vllm.cpp, LocalAI's C++ vLLM-style
+ runtime. The published checkpoint generates on CUDA and measured 16.7
+ tokens/s on GB10 with the pinned revision and the limits configured here.
+
+ This is the target-only setup. Use the DFlash2 variant for the measured
+ speculative-decoding configuration. The entry downloads the complete
+ revision-pinned repository, including its configuration, tokenizer, index,
+ and safetensors shards.
+ tags:
+ - llm
+ - qwen
+ - qwen3.8
+ - exl3
+ - vllm-cpp
+ - reasoning
+ - tool-calling
+ - gpu
+ - cuda
+ size: 15.4GB
+ last_checked: "2026-09-07"
+ overrides:
+ backend: vllm-cpp
+ known_usecases:
+ - chat
+ - completion
+ function:
+ grammar:
+ disable: true
+ template:
+ use_tokenizer_template: true
+ context_size: 8192
+ engine_args:
+ num_blocks: 2048
+ max_num_seqs: 8
+ max_num_batched_tokens: 16384
+ enable_prefix_caching: false
+ parameters:
+ model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ artifacts:
+ - name: model
+ target: model
+ source:
+ type: huggingface
+ repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ revision: 19441ac874c4018295da848e250f23511361cda4
+- name: qwen3.8-27b-dflash2-exl3-vllm-cpp
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ - https://huggingface.co/Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+ - https://github.com/mudler/vllm.cpp
+ description: |
+ Qwen3.8-27B EXL3 with its EXL3 DFlash2 companion, served by vllm.cpp. On
+ GB10 this pinned pair measured 48.7 tokens/s at a seven-token draft budget,
+ versus 16.7 tokens/s target-only, with token-identical greedy output.
+
+ LocalAI stages both complete Hugging Face repositories before load. The
+ content-addressed companion snapshot is passed to the backend as the draft
+ model, while the speculative method and seven-token budget remain fixed.
+ tags:
+ - llm
+ - qwen
+ - qwen3.8
+ - exl3
+ - vllm-cpp
+ - speculative-decoding
+ - dflash
+ - reasoning
+ - tool-calling
+ - gpu
+ - cuda
+ size: 16.9GB
+ last_checked: "2026-09-07"
+ variants:
+ - model: qwen3.8-27b-exl3-vllm-cpp
+ overrides:
+ backend: vllm-cpp
+ known_usecases:
+ - chat
+ - completion
+ function:
+ grammar:
+ disable: true
+ template:
+ use_tokenizer_template: true
+ context_size: 8192
+ engine_args:
+ num_blocks: 2048
+ max_num_seqs: 8
+ max_num_batched_tokens: 16384
+ enable_prefix_caching: false
+ speculative_config:
+ method: dflash
+ model: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+ num_speculative_tokens: 7
+ parameters:
+ model: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ artifacts:
+ - name: model
+ target: model
+ source:
+ type: huggingface
+ repo: Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw
+ revision: 19441ac874c4018295da848e250f23511361cda4
+ - name: draft_model
+ target: companion
+ source:
+ type: huggingface
+ repo: Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw
+ revision: 4f0436269bca761b071f05319e8e04a87cc633f9
+- name: deepseek-v4-flash-spark-exl3-vllm-cpp
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/0xSero/deepseek-v4-flash-0731-spark
+ - https://github.com/mudler/vllm.cpp
+ description: |
+ DeepSeek V4 Flash's Spark and GB10-oriented REAP-K216 EXL3 checkpoint,
+ served by vllm.cpp. It needs CUDA and roughly 100 GiB for its large
+ rank-sliced checkpoint.
+
+ The repository is pinned to its latest recorded revision. vllm.cpp's
+ existing runtime evidence measured the older 22f28d32b9b29b4352eaa380ff8c2c170b2847ab
+ revision; this entry does not claim that the newer revision has passed the
+ same end-to-end gate.
+ tags:
+ - llm
+ - deepseek
+ - deepseek-v4
+ - exl3
+ - vllm-cpp
+ - gpu
+ - cuda
+ - gb10
+ size: 100GB
+ last_checked: "2026-09-07"
+ overrides:
+ backend: vllm-cpp
+ known_usecases:
+ - chat
+ - completion
+ template:
+ use_tokenizer_template: true
+ parameters:
+ model: 0xSero/deepseek-v4-flash-0731-spark
+ artifacts:
+ - name: model
+ target: model
+ source:
+ type: huggingface
+ repo: 0xSero/deepseek-v4-flash-0731-spark
+ revision: ce5ff0f1efb2e184aafc759d281bfae47d3a359c
+- name: deepseek-v4-flash-exl3-3bpw-vllm-cpp
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+ - https://github.com/mudler/vllm.cpp
+ description: |
+ Experimental non-Spark DeepSeek V4 Flash EXL3 3.0bpw checkpoint served by
+ vllm.cpp. This is the complete, non-REAP layout and requires a large
+ multi-GPU CUDA system.
+
+ The publisher describes the artifact as structurally complete but has not
+ passed end-to-end generation. Treat this entry as an integration target,
+ not as a correctness- or performance-gated configuration.
+ tags:
+ - llm
+ - deepseek
+ - deepseek-v4
+ - exl3
+ - vllm-cpp
+ - experimental
+ - gpu
+ - cuda
+ last_checked: "2026-09-07"
+ overrides:
+ backend: vllm-cpp
+ known_usecases:
+ - chat
+ - completion
+ template:
+ use_tokenizer_template: true
+ parameters:
+ model: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+ artifacts:
+ - name: model
+ target: model
+ source:
+ type: huggingface
+ repo: 0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw
+ revision: e0bf84ac76a5100e8790c22ad10b70b1e2d06d71
- name: qwen3.6-27b-nvfp4-vllm-cpp
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:
@@ -16456,6 +18030,76 @@
- filename: mmproj-Qwen3-Omni-30B-A3B-Thinking-Q8_0.gguf
sha256: 2bd5459571f8230a0c251d3d0dd36267753f0800ed145449a34f220a31f93898
uri: huggingface://ggml-org/Qwen3-Omni-30B-A3B-Thinking-GGUF/mmproj-Qwen3-Omni-30B-A3B-Thinking-Q8_0.gguf
+- &lightonocr-2-1b
+ name: "lightonocr-2-1b"
+ variants:
+ - model: lightonocr-2-1b-f16
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/lightonai/LightOnOCR-2-1B
+ - https://huggingface.co/ggml-org/LightOnOCR-2-1B-GGUF
+ description: |
+ LightOnOCR-2-1B is a compact Apache-2.0 vision-language model for optical
+ character recognition and document understanding. It extracts text,
+ tables, forms, and structured content from images and PDFs in multiple
+ languages. This default entry uses the Q8_0 model and vision projector.
+ license: apache-2.0
+ icon: https://huggingface.co/lightonai.png
+ tags:
+ - llm
+ - gguf
+ - gpu
+ - image-to-text
+ - ocr
+ - multimodal
+ - cpu
+ - mistral
+ last_checked: "2026-08-13"
+ overrides:
+ backend: llama-cpp
+ known_usecases:
+ - chat
+ - vision
+ mmproj: mmproj-LightOnOCR-2-1B-Q8_0.gguf
+ options:
+ - use_jinja:true
+ parameters:
+ model: LightOnOCR-2-1B-Q8_0.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: LightOnOCR-2-1B-Q8_0.gguf
+ sha256: f4cfc0ea9765a4cac22a235191c48aefa94df223419f5ab658b6201f36206d58
+ uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/LightOnOCR-2-1B-Q8_0.gguf
+ - filename: mmproj-LightOnOCR-2-1B-Q8_0.gguf
+ sha256: e9b45eb85b0f0afb3cc138236eec7adb75879420130e5e2049ee9af3f24e337f
+ uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/mmproj-LightOnOCR-2-1B-Q8_0.gguf
+- !!merge <<: *lightonocr-2-1b
+ name: "lightonocr-2-1b-f16"
+ variants: []
+ description: |
+ LightOnOCR-2-1B F16 is the full-precision GGUF build for optical character
+ recognition and multilingual document understanding. It pairs the F16
+ language model with the matching F16 vision projector.
+ overrides:
+ backend: llama-cpp
+ known_usecases:
+ - chat
+ - vision
+ mmproj: mmproj-LightOnOCR-2-1B-f16.gguf
+ options:
+ - use_jinja:true
+ parameters:
+ model: LightOnOCR-2-1B-f16.gguf
+ template:
+ use_tokenizer_template: true
+ files:
+ - filename: LightOnOCR-2-1B-f16.gguf
+ sha256: 83f5b7a24410de69390119dd18e7866ecd0687564423846fc344ed3a46647a0b
+ uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/LightOnOCR-2-1B-f16.gguf
+ - filename: mmproj-LightOnOCR-2-1B-f16.gguf
+ sha256: 1c4a3a27f7c5aa90b455ed5ec6dc79c9b9746739a8d840022790414376738130
+ uri: huggingface://ggml-org/LightOnOCR-2-1B-GGUF/mmproj-LightOnOCR-2-1B-f16.gguf
- name: glm-ocr
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:
@@ -54879,6 +56523,49 @@
- model: gemma-4-26b-a4b-it-heretic-apex-i-balanced
- model: gemma-4-26b-a4b-it-heretic-apex-i-compact
- model: gemma-4-26b-a4b-it-heretic-apex-i-mini
+- name: audio-cpp-indextts-2.5
+ url: github:mudler/LocalAI/gallery/virtual.yaml@master
+ urls:
+ - https://huggingface.co/IndexTeam/IndexTTS-2.5
+ - https://huggingface.co/Richasy/IndexTTS-2.5-GGUF
+ - https://github.com/0xShug0/audio.cpp
+ description: |
+ IndexTTS 2.5 is a multilingual text-to-speech model with zero-shot voice
+ cloning and emotion control, served by the audio-cpp backend. Supply a
+ server-local WAV path in the OpenAI `voice` field as the speaker reference.
+
+ This entry uses the published original-dtype GGUF. The package preserves
+ the source tensors without another quantization step, and audio.cpp records
+ the original-dtype IndexTTS2 path as passing its validation suite.
+
+ The bilibili Model Use License governs the model. Review its use,
+ redistribution, and large-scale commercial deployment terms before
+ installation.
+ license: other
+ tags:
+ - audio-cpp
+ - indextts
+ - multilingual
+ - tts
+ - text-to-speech
+ - voice-cloning
+ - emotion-control
+ - gguf
+ - ggml
+ last_checked: "2026-08-15"
+ overrides:
+ backend: audio-cpp
+ known_usecases:
+ - tts
+ name: audio-cpp-indextts-2.5
+ options:
+ - backend:best
+ parameters:
+ model: audio-cpp/index-tts2_5-orig.gguf
+ files:
+ - filename: audio-cpp/index-tts2_5-orig.gguf
+ sha256: 07e9bfe77bd42b6e67b8e5a39b365635f7198951281990cb49f322ef546cb9b4
+ uri: huggingface://Richasy/IndexTTS-2.5-GGUF/IndexTTS2.5-GGUF/index-tts2_5-orig.gguf
- name: audio-cpp-supertonic
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:
@@ -54910,6 +56597,8 @@
known_usecases:
- tts
name: audio-cpp-supertonic
+ options:
+ - backend:best
parameters:
model: audio-cpp/supertonic-3-orig.gguf
files:
@@ -54952,6 +56641,8 @@
known_usecases:
- tts
name: audio-cpp-higgs-audio-v3
+ options:
+ - backend:best
parameters:
model: audio-cpp/higgs-audio-v3-tts-4b-q8_0.gguf
files:
@@ -55001,6 +56692,8 @@
- tts
- audio_transform
name: audio-cpp-chatterbox
+ options:
+ - backend:best
parameters:
model: audio-cpp/chatterbox-q8_0.gguf
files:
@@ -55037,6 +56730,8 @@
known_usecases:
- transcript
name: audio-cpp-citrinet-asr
+ options:
+ - backend:best
parameters:
model: audio-cpp/citrinet-asr-q8_0.gguf
files:
@@ -55073,6 +56768,8 @@
known_usecases:
- transcript
name: audio-cpp-nemotron-asr
+ options:
+ - backend:best
parameters:
model: audio-cpp/nemotron-3.5-asr-streaming-0.6b-f16.gguf
files:
@@ -55110,6 +56807,8 @@
known_usecases:
- diarization
name: audio-cpp-sortformer-diarization
+ options:
+ - backend:best
parameters:
model: audio-cpp/sortformer-diar-4spk-v1-q8_0.gguf
files:
@@ -55147,6 +56846,8 @@
known_usecases:
- audio_transform
name: audio-cpp-htdemucs
+ options:
+ - backend:best
parameters:
model: audio-cpp/htdemucs-f16.gguf
files:
@@ -55186,6 +56887,8 @@
known_usecases:
- sound_generation
name: audio-cpp-stable-audio-sfx
+ options:
+ - backend:best
parameters:
model: audio-cpp/stable-audio-3-small-sfx-q8_0.gguf
files:
@@ -55224,6 +56927,8 @@
known_usecases:
- transcript
name: audio-cpp-forced-aligner
+ options:
+ - backend:best
parameters:
language: en
model: audio-cpp/qwen3-forced-aligner-0.6b-q8_0.gguf
@@ -55253,6 +56958,7 @@
- vad
name: audio-cpp-silero-vad
options:
+ - backend:best
- family:silero_vad
parameters:
model: bundled:silero_vad
@@ -55280,6 +56986,7 @@
- vad
name: audio-cpp-marblenet-vad
options:
+ - backend:best
- family:marblenet_vad
parameters:
model: bundled:marblenet_vad
@@ -55319,6 +57026,8 @@
known_usecases:
- tts
name: audio-cpp-irodori-voicedesign
+ options:
+ - backend:best
parameters:
model: audio-cpp/irodori-tts-600m-v3-voicedesign-q8_0.gguf
files:
@@ -55363,6 +57072,7 @@
- audio_transform
name: audio-cpp-seedvc-singing
options:
+ - backend:best
- task:svc
parameters:
model: audio-cpp/seed-vc-mlx-q8_0.gguf
@@ -55415,6 +57125,7 @@
- audio_transform
name: audio-cpp-vevo2-speech-to-speech
options:
+ - backend:best
- task:s2s
parameters:
model: audio-cpp/vevo2-q8_0.gguf
diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go
index e73d45535..49276abc3 100644
--- a/pkg/model/initializers.go
+++ b/pkg/model/initializers.go
@@ -173,7 +173,7 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
if !ready {
xlog.Debug("GRPC Service NOT ready")
startupErr := grpcStartupError(client.Process())
- stopLoadProcess(client, modelID)
+ ml.stopLoadProcess(client, modelID)
return nil, startupErr
}
@@ -189,11 +189,11 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, options)
if err != nil {
- stopLoadProcess(client, modelID)
+ ml.stopLoadProcess(client, modelID)
return nil, fmt.Errorf("could not load model: %w", err)
}
if !res.Success {
- stopLoadProcess(client, modelID)
+ ml.stopLoadProcess(client, modelID)
return nil, fmt.Errorf("could not load model (no success): %s", res.Message)
}
@@ -260,7 +260,7 @@ func lastNonEmptyLine(path string, maxBytes int64) string {
// stopLoadProcess tears down a backend process whose load did not complete.
// The stop error is only logged: the load error is what the caller reports.
-func stopLoadProcess(client *Model, modelID string) {
+func (ml *ModelLoader) stopLoadProcess(client *Model, modelID string) {
process := client.Process()
if process == nil {
return
@@ -268,6 +268,7 @@ func stopLoadProcess(client *Model, modelID string) {
if err := process.Stop(); err != nil {
xlog.Warn("failed to stop backend process after failed load", "error", err, "modelID", modelID)
}
+ ml.cleanupProcessRuntime(process)
}
// parallelSlotsFromOptions returns the effective n_parallel from the backend
diff --git a/pkg/model/loader.go b/pkg/model/loader.go
index 322b11e36..2df4aee2a 100644
--- a/pkg/model/loader.go
+++ b/pkg/model/loader.go
@@ -106,6 +106,10 @@ type ModelLoader struct {
// the exit code can't, since a child killed by our own SIGTERM/SIGKILL
// reports -1, indistinguishable from a signal-induced crash.
stoppingProcs sync.Map
+ // processRuntimes keeps the owned state/scratch directory alive until the
+ // loader has consumed any exit diagnostics. The exit watcher removes the
+ // potentially large scratch contents immediately.
+ processRuntimes sync.Map
// loadFailures records, per modelID, the cooldown window applied after a
// failed load so that a client repeatedly polling a broken model does not
// spawn (and leak) a fresh backend process on every request. Guarded by mu.
diff --git a/pkg/model/process.go b/pkg/model/process.go
index fb1ed004a..9b799e5a5 100644
--- a/pkg/model/process.go
+++ b/pkg/model/process.go
@@ -177,12 +177,14 @@ func (ml *ModelLoader) deleteProcess(ctx context.Context, s string, force bool)
// A concurrently crashed/already-reaped process can no longer own
// resources even if Stop could not read or signal its PID.
store.Delete(s)
+ ml.cleanupProcessRuntime(process)
return nil
}
return err
}
store.Delete(s)
+ ml.cleanupProcessRuntime(process)
return nil
}
func (ml *ModelLoader) StopGRPC(filter GRPCProcessFilter) error {
@@ -231,16 +233,6 @@ func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string
return ml.startProcess(grpcProcess, id, serverAddress, args...)
}
-// newProcessStateDir creates the directory a backend process uses for its pid,
-// state and log files, and reports why when it cannot.
-func newProcessStateDir() (string, error) {
- dir, err := os.MkdirTemp(os.TempDir(), "go-processmanager")
- if err != nil {
- return "", fmt.Errorf("creating backend process state directory under %s: %w", os.TempDir(), err)
- }
- return dir, nil
-}
-
func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) {
// Make sure the process is executable
// Check first if it has executable permissions
@@ -262,7 +254,12 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
return nil, err
}
- env := os.Environ()
+ runtime, err := newBackendProcessRuntime()
+ if err != nil {
+ return nil, err
+ }
+
+ env := backendTempEnvironment(os.Environ(), runtime.tempDir)
// Vulkan backends are self-contained: they bundle their own loader and
// Mesa driver .so files in lib/ plus the matching ICD manifests in
// vulkan/icd.d/. Point the loader at those manifests so it doesn't rely on
@@ -271,16 +268,14 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
// and the GPU would silently fall back to CPU). No-op for other backends.
env = append(env, vulkanICDEnv(workDir)...)
- // Resolve the state directory here rather than through
+ // Resolve and own the state directory here rather than through
// process.WithTemporaryStateDir(). process.New applies its options but
// discards the error they return, so a temp directory that cannot be
// created leaves StateDir empty and every later option unapplied. Run()
- // then reported "mkdir : no such file or directory" with no path, hiding
- // the real cause (a full volume, or a TMPDIR that no longer resolves).
- stateDir, err := newProcessStateDir()
- if err != nil {
- return nil, err
- }
+ // then reports "mkdir : no such file or directory" with no useful path.
+ // The same owned directory also contains backend scratch so an unexpected
+ // exit cannot strand request files directly in the host's shared /tmp.
+ stateDir := runtime.dir
grpcControlProcess := process.New(
process.WithStateDir(stateDir),
@@ -296,8 +291,10 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
}
if err := grpcControlProcess.Run(); err != nil {
+ runtime.cleanup()
return grpcControlProcess, err
}
+ ml.processRuntimes.Store(grpcControlProcess, runtime)
xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir())
@@ -376,11 +373,35 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
}
xlog.Warn("Backend process exited unexpectedly", fields...)
}
+ runtime.cleanupScratch()
+ close(runtime.diagnosticsDone)
}()
return grpcControlProcess, nil
}
+func (ml *ModelLoader) cleanupProcessRuntime(process *process.Process) {
+ if process == nil {
+ return
+ }
+ value, ok := ml.processRuntimes.LoadAndDelete(process)
+ if !ok {
+ return
+ }
+ runtime := value.(*backendProcessRuntime)
+ go func() {
+ <-runtime.diagnosticsDone
+ runtime.cleanup()
+ }()
+}
+
+// CleanupProcessRuntime releases state and scratch owned by a process started
+// through StartProcess. Callers that supervise processes outside ModelLoader's
+// model store must invoke it after they have consumed exit diagnostics.
+func (ml *ModelLoader) CleanupProcessRuntime(process *process.Process) {
+ ml.cleanupProcessRuntime(process)
+}
+
// vulkanICDEnv returns environment overrides that point the Vulkan loader at
// the ICD manifests a backend bundles in /vulkan/icd.d. Vulkan
// backends ship a self-contained stack — their own loader and Mesa driver .so
diff --git a/pkg/model/process_exit_test.go b/pkg/model/process_exit_test.go
index cc5554bcf..2b941bf6e 100644
--- a/pkg/model/process_exit_test.go
+++ b/pkg/model/process_exit_test.go
@@ -15,8 +15,10 @@ import (
var _ = Describe("backend process exit diagnostics", func() {
It("includes the exit code and final stderr line for an unexpected exit", func() {
tmpDir := GinkgoT().TempDir()
+ backendTempRoot := filepath.Join(tmpDir, "backend-runtime")
+ GinkgoT().Setenv(backendTempDirEnv, backendTempRoot)
backendPath := filepath.Join(tmpDir, "failing-backend")
- Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\necho 'first diagnostic' >&2\necho 'fatal metal pipeline error' >&2\nexit 42\n"), 0o700)).To(Succeed())
+ Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\nprintf '%s' \"$TMPDIR\" > \"$0.tmpdir\"\necho 'first diagnostic' >&2\necho 'fatal metal pipeline error' >&2\nexit 42\n"), 0o700)).To(Succeed())
captured := &bytes.Buffer{}
handler := slog.NewTextHandler(captured, &slog.HandlerOptions{Level: slog.LevelWarn})
@@ -29,10 +31,16 @@ var _ = Describe("backend process exit diagnostics", func() {
process, err := loader.startProcess(backendPath, "test-model", "127.0.0.1:65535")
Expect(err).ToNot(HaveOccurred())
Eventually(process.Done()).Should(BeClosed())
+ backendTemp, err := os.ReadFile(backendPath + ".tmpdir")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(string(backendTemp)).To(Equal(filepath.Join(process.StateDir(), "tmp")))
+ Eventually(string(backendTemp)).ShouldNot(BeADirectory())
Eventually(captured.String).Should(And(
ContainSubstring("Backend process exited unexpectedly"),
ContainSubstring("exitCode=42"),
ContainSubstring(`stderr="fatal metal pipeline error"`),
))
+ loader.cleanupProcessRuntime(process)
+ Eventually(process.StateDir()).ShouldNot(BeADirectory())
})
})
diff --git a/pkg/model/process_runtime.go b/pkg/model/process_runtime.go
new file mode 100644
index 000000000..b7443ef7d
--- /dev/null
+++ b/pkg/model/process_runtime.go
@@ -0,0 +1,168 @@
+package model
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "github.com/gofrs/flock"
+ "github.com/mudler/xlog"
+)
+
+const (
+ backendTempDirEnv = "LOCALAI_BACKEND_TEMP_DIR"
+ backendRuntimeDirPrefix = "process-"
+ backendRuntimeMarker = ".localai-backend-runtime"
+ backendRuntimeMagic = "localai-backend-runtime-v1\n"
+)
+
+// backendProcessRuntime owns both go-processmanager's state and all temporary
+// files created by one backend process. The held lock distinguishes a live
+// runtime from one abandoned when LocalAI was killed or crashed.
+type backendProcessRuntime struct {
+ dir string
+ tempDir string
+ lock *flock.Flock
+ scratch sync.Once
+ once sync.Once
+ // diagnosticsDone closes after the exit watcher has read the state files.
+ diagnosticsDone chan struct{}
+}
+
+func backendRuntimeRoot() string {
+ base := os.TempDir()
+ if configured := os.Getenv(backendTempDirEnv); configured != "" {
+ base = configured
+ }
+ // Always append a LocalAI- and user-specific namespace. Even if an operator
+ // points the configurable base at /tmp, the sweeper never inspects unrelated
+ // process-* directories in that shared parent.
+ return filepath.Join(base, fmt.Sprintf("localai-%d", os.Getuid()), "backend-runtime")
+}
+
+func newBackendProcessRuntime() (*backendProcessRuntime, error) {
+ root := backendRuntimeRoot()
+ if err := os.MkdirAll(root, 0o700); err != nil {
+ return nil, fmt.Errorf("creating backend runtime root %s: %w", root, err)
+ }
+
+ // Serialize sweeping with creation. Otherwise a second LocalAI instance
+ // could observe the new directory in the tiny window before its owner lock
+ // is acquired and mistake it for an abandoned runtime.
+ sweepLock := flock.New(filepath.Join(root, ".sweep.lock"))
+ if err := sweepLock.Lock(); err != nil {
+ return nil, fmt.Errorf("locking backend runtime root %s: %w", root, err)
+ }
+ defer func() {
+ if err := sweepLock.Unlock(); err != nil {
+ xlog.Warn("Failed to unlock backend runtime root", "root", root, "error", err)
+ }
+ }()
+
+ sweepAbandonedBackendRuntimes(root)
+
+ dir, err := os.MkdirTemp(root, backendRuntimeDirPrefix)
+ if err != nil {
+ return nil, fmt.Errorf("creating backend process runtime under %s: %w", root, err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, backendRuntimeMarker), []byte(backendRuntimeMagic), 0o600); err != nil {
+ _ = os.RemoveAll(dir)
+ return nil, fmt.Errorf("marking backend process runtime %s: %w", dir, err)
+ }
+ runtimeLock := flock.New(filepath.Join(dir, ".owner.lock"))
+ if err := runtimeLock.Lock(); err != nil {
+ _ = os.RemoveAll(dir)
+ return nil, fmt.Errorf("locking backend process runtime %s: %w", dir, err)
+ }
+ tempDir := filepath.Join(dir, "tmp")
+ if err := os.Mkdir(tempDir, 0o700); err != nil {
+ _ = runtimeLock.Unlock()
+ _ = os.RemoveAll(dir)
+ return nil, fmt.Errorf("creating backend scratch directory %s: %w", tempDir, err)
+ }
+
+ return &backendProcessRuntime{
+ dir: dir,
+ tempDir: tempDir,
+ lock: runtimeLock,
+ diagnosticsDone: make(chan struct{}),
+ }, nil
+}
+
+func sweepAbandonedBackendRuntimes(root string) {
+ entries, err := os.ReadDir(root)
+ if err != nil {
+ xlog.Warn("Failed to inspect backend runtime root", "root", root, "error", err)
+ return
+ }
+ for _, entry := range entries {
+ if !entry.IsDir() || !strings.HasPrefix(entry.Name(), backendRuntimeDirPrefix) {
+ continue
+ }
+ dir := filepath.Join(root, entry.Name())
+ marker, err := os.ReadFile(filepath.Join(dir, backendRuntimeMarker))
+ if err != nil || string(marker) != backendRuntimeMagic {
+ continue
+ }
+ ownerLock := flock.New(filepath.Join(dir, ".owner.lock"))
+ available, err := ownerLock.TryLock()
+ if err != nil {
+ xlog.Warn("Failed to inspect backend runtime ownership", "dir", dir, "error", err)
+ continue
+ }
+ if !available {
+ continue
+ }
+ if err := ownerLock.Unlock(); err != nil {
+ xlog.Warn("Failed to release abandoned backend runtime lock", "dir", dir, "error", err)
+ continue
+ }
+ if err := os.RemoveAll(dir); err != nil {
+ xlog.Warn("Failed to remove abandoned backend runtime", "dir", dir, "error", err)
+ }
+ }
+}
+
+func (r *backendProcessRuntime) cleanup() {
+ if r == nil {
+ return
+ }
+ r.once.Do(func() {
+ r.cleanupScratch()
+ if err := r.lock.Unlock(); err != nil {
+ xlog.Warn("Failed to unlock backend process runtime", "dir", r.dir, "error", err)
+ }
+ if err := os.RemoveAll(r.dir); err != nil {
+ xlog.Warn("Failed to remove backend process runtime", "dir", r.dir, "error", err)
+ }
+ })
+}
+
+func (r *backendProcessRuntime) cleanupScratch() {
+ if r == nil {
+ return
+ }
+ r.scratch.Do(func() {
+ if err := os.RemoveAll(r.tempDir); err != nil {
+ xlog.Warn("Failed to remove backend scratch directory", "dir", r.tempDir, "error", err)
+ }
+ })
+}
+
+func backendTempEnvironment(env []string, tempDir string) []string {
+ result := make([]string, 0, len(env)+3)
+ for _, entry := range env {
+ key, _, found := strings.Cut(entry, "=")
+ if found && (key == "TMPDIR" || key == "TMP" || key == "TEMP") {
+ continue
+ }
+ result = append(result, entry)
+ }
+ return append(result,
+ "TMPDIR="+tempDir,
+ "TMP="+tempDir,
+ "TEMP="+tempDir,
+ )
+}
diff --git a/pkg/model/process_runtime_test.go b/pkg/model/process_runtime_test.go
new file mode 100644
index 000000000..b27547300
--- /dev/null
+++ b/pkg/model/process_runtime_test.go
@@ -0,0 +1,109 @@
+package model
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Backend process runtime directory", func() {
+ It("keeps active runtimes while sweeping abandoned ones", func() {
+ root := GinkgoT().TempDir()
+ GinkgoT().Setenv(backendTempDirEnv, root)
+ ownedRoot := backendRuntimeRoot()
+ unrelated := filepath.Join(root, backendRuntimeDirPrefix+"unrelated")
+ Expect(os.MkdirAll(unrelated, 0o700)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(unrelated, "keep"), []byte("unrelated"), 0o600)).To(Succeed())
+
+ active, err := newBackendProcessRuntime()
+ Expect(err).ToNot(HaveOccurred())
+ DeferCleanup(active.cleanup)
+ Expect(os.WriteFile(filepath.Join(active.tempDir, "active.img"), []byte("active"), 0o600)).To(Succeed())
+
+ abandoned := filepath.Join(ownedRoot, backendRuntimeDirPrefix+"abandoned")
+ Expect(os.MkdirAll(abandoned, 0o700)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(abandoned, backendRuntimeMarker), []byte(backendRuntimeMagic), 0o600)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(abandoned, "orphan.img"), []byte("orphan"), 0o600)).To(Succeed())
+ foreign := filepath.Join(ownedRoot, backendRuntimeDirPrefix+"foreign")
+ Expect(os.MkdirAll(foreign, 0o700)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(foreign, "keep"), []byte("foreign"), 0o600)).To(Succeed())
+
+ other, err := newBackendProcessRuntime()
+ Expect(err).ToNot(HaveOccurred())
+ DeferCleanup(other.cleanup)
+
+ Expect(active.dir).To(BeADirectory())
+ Expect(abandoned).ToNot(BeAnExistingFile())
+ Expect(foreign).To(BeADirectory())
+ Expect(unrelated).To(BeADirectory())
+ })
+
+ It("uses one private directory for process state and backend scratch", func() {
+ root := GinkgoT().TempDir()
+ GinkgoT().Setenv(backendTempDirEnv, root)
+
+ runtime, err := newBackendProcessRuntime()
+ Expect(err).ToNot(HaveOccurred())
+ DeferCleanup(runtime.cleanup)
+
+ Expect(filepath.Dir(runtime.dir)).To(Equal(backendRuntimeRoot()))
+ Expect(runtime.tempDir).To(Equal(filepath.Join(runtime.dir, "tmp")))
+ info, err := os.Stat(runtime.tempDir)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(info.IsDir()).To(BeTrue())
+ Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o700)))
+ })
+
+ It("overrides inherited temp variables for the backend only", func() {
+ env := backendTempEnvironment([]string{
+ "PATH=/bin",
+ "TMPDIR=/old/tmpdir",
+ "TMP=/old/tmp",
+ "TEMP=/old/temp",
+ }, "/owned/scratch")
+
+ Expect(env).To(ConsistOf(
+ "PATH=/bin",
+ "TMPDIR=/owned/scratch",
+ "TMP=/owned/scratch",
+ "TEMP=/owned/scratch",
+ ))
+ for _, key := range []string{"TMPDIR", "TMP", "TEMP"} {
+ count := 0
+ for _, entry := range env {
+ if strings.HasPrefix(entry, key+"=") {
+ count++
+ }
+ }
+ Expect(count).To(Equal(1), key)
+ }
+ })
+
+ It("removes the runtime when its owner exits", func() {
+ GinkgoT().Setenv(backendTempDirEnv, GinkgoT().TempDir())
+
+ runtime, err := newBackendProcessRuntime()
+ Expect(err).ToNot(HaveOccurred())
+ dir := runtime.dir
+ runtime.cleanup()
+
+ Expect(dir).ToNot(BeAnExistingFile())
+ })
+
+ It("reports which configured root cannot be used", func() {
+ parent := GinkgoT().TempDir()
+ file := filepath.Join(parent, "not-a-directory")
+ Expect(os.WriteFile(file, []byte("x"), 0o600)).To(Succeed())
+ base := filepath.Join(file, "backend-runtime")
+ GinkgoT().Setenv(backendTempDirEnv, base)
+ root := backendRuntimeRoot()
+
+ runtime, err := newBackendProcessRuntime()
+ Expect(err).To(HaveOccurred())
+ Expect(runtime).To(BeNil())
+ Expect(err.Error()).To(ContainSubstring(root))
+ })
+})
diff --git a/pkg/model/process_statedir_test.go b/pkg/model/process_statedir_test.go
deleted file mode 100644
index 207c2991d..000000000
--- a/pkg/model/process_statedir_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package model
-
-import (
- "os"
- "path/filepath"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
-)
-
-var _ = Describe("Backend process state directory", func() {
- It("reports why the state directory could not be created", func() {
- // A worker whose volume is full, or whose TMPDIR no longer resolves,
- // cannot get a state directory. go-processmanager's New() drops the
- // option error, leaving StateDir empty, and Run() then failed with
- // "mkdir : no such file or directory" naming no path at all. Resolving
- // the directory here keeps the real cause attached.
- GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "does-not-exist"))
-
- dir, err := newProcessStateDir()
- Expect(err).To(HaveOccurred())
- Expect(dir).To(BeEmpty())
- Expect(err.Error()).To(ContainSubstring("backend process state directory"))
- Expect(err.Error()).To(ContainSubstring("does-not-exist"),
- "the error must name the directory it could not create")
- })
-
- It("returns a usable directory when the temp location works", func() {
- GinkgoT().Setenv("TMPDIR", GinkgoT().TempDir())
-
- dir, err := newProcessStateDir()
- Expect(err).ToNot(HaveOccurred())
- Expect(dir).ToNot(BeEmpty())
- info, statErr := os.Stat(dir)
- Expect(statErr).ToNot(HaveOccurred())
- Expect(info.IsDir()).To(BeTrue())
- })
-})
diff --git a/pkg/safefile/remove_other.go b/pkg/safefile/remove_other.go
new file mode 100644
index 000000000..1fed6a21d
--- /dev/null
+++ b/pkg/safefile/remove_other.go
@@ -0,0 +1,17 @@
+//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris
+
+package safefile
+
+import (
+ "errors"
+ "fmt"
+)
+
+// ErrUnsafePath reports a path shape or file type that exact removal refuses.
+var ErrUnsafePath = errors.New("unsafe removal path")
+
+// RemoveExact fails closed on platforms without component-relative no-follow
+// filesystem operations.
+func RemoveExact(root, relativePath string, sidecarSuffixes []string, pruneParents int) error {
+ return fmt.Errorf("secure exact removal is unsupported on this platform")
+}
diff --git a/pkg/safefile/remove_unix.go b/pkg/safefile/remove_unix.go
new file mode 100644
index 000000000..9509e1e19
--- /dev/null
+++ b/pkg/safefile/remove_unix.go
@@ -0,0 +1,144 @@
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package safefile
+
+import (
+ "errors"
+ "fmt"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/sys/unix"
+)
+
+// ErrUnsafePath reports a path shape or file type that exact removal refuses.
+var ErrUnsafePath = errors.New("unsafe removal path")
+
+// RemoveExact removes a file and its named sidecars below root without
+// following symbolic links. It prunes up to pruneParents empty parent
+// directories, but never removes root itself.
+func RemoveExact(root, relativePath string, sidecarSuffixes []string, pruneParents int) error {
+ return removeExact(root, relativePath, sidecarSuffixes, pruneParents, nil)
+}
+
+func removeExact(root, relativePath string, sidecarSuffixes []string, pruneParents int, parentsOpened func()) error {
+ parts, err := cleanRelativeParts(relativePath)
+ if err != nil {
+ return err
+ }
+ if pruneParents < 0 {
+ return fmt.Errorf("%w: prune parent count must not be negative", ErrUnsafePath)
+ }
+
+ rootFD, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
+ if err != nil {
+ return fmt.Errorf("opening removal root %q: %w", root, err)
+ }
+ handles := []int{rootFD}
+ defer func() {
+ for i := len(handles) - 1; i >= 0; i-- {
+ _ = unix.Close(handles[i])
+ }
+ }()
+
+ for _, component := range parts[:len(parts)-1] {
+ fd, openErr := unix.Openat(handles[len(handles)-1], component, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
+ if errors.Is(openErr, unix.ENOENT) {
+ return nil
+ }
+ if openErr != nil {
+ if errors.Is(openErr, unix.ELOOP) || errors.Is(openErr, unix.ENOTDIR) {
+ return fmt.Errorf("%w: path component %q is not a directory: %v", ErrUnsafePath, component, openErr)
+ }
+ return fmt.Errorf("opening removal path component %q: %w", component, openErr)
+ }
+ handles = append(handles, fd)
+ }
+ if parentsOpened != nil {
+ parentsOpened()
+ }
+
+ parentFD := handles[len(handles)-1]
+ leaf := parts[len(parts)-1]
+ names := make([]string, 0, len(sidecarSuffixes)+1)
+ names = append(names, leaf)
+ for _, suffix := range sidecarSuffixes {
+ if suffix == "" || strings.ContainsAny(suffix, `/\\`) {
+ return fmt.Errorf("%w: invalid sidecar suffix %q", ErrUnsafePath, suffix)
+ }
+ names = append(names, leaf+suffix)
+ }
+
+ for _, name := range names {
+ var stat unix.Stat_t
+ statErr := unix.Fstatat(parentFD, name, &stat, unix.AT_SYMLINK_NOFOLLOW)
+ if errors.Is(statErr, unix.ENOENT) {
+ continue
+ }
+ if statErr != nil {
+ return fmt.Errorf("stating removal entry %q: %w", name, statErr)
+ }
+ switch stat.Mode & unix.S_IFMT {
+ case unix.S_IFLNK:
+ return fmt.Errorf("%w: refusing to remove symbolic link %q", ErrUnsafePath, name)
+ case unix.S_IFDIR:
+ return fmt.Errorf("%w: release key identifies a directory", ErrUnsafePath)
+ }
+ }
+
+ for _, name := range names {
+ if unlinkErr := unix.Unlinkat(parentFD, name, 0); unlinkErr != nil && !errors.Is(unlinkErr, unix.ENOENT) {
+ return fmt.Errorf("removing entry %q: %w", name, unlinkErr)
+ }
+ }
+
+ maxPrune := min(pruneParents, len(handles)-1)
+ for childIndex := len(handles) - 1; childIndex >= len(handles)-maxPrune; childIndex-- {
+ parentIndex := childIndex - 1
+ name := parts[childIndex-1]
+ same, identityErr := sameDirectoryEntry(handles[parentIndex], name, handles[childIndex])
+ if identityErr != nil {
+ if errors.Is(identityErr, unix.ENOENT) {
+ break
+ }
+ return fmt.Errorf("checking directory %q before pruning: %w", name, identityErr)
+ }
+ if !same {
+ break
+ }
+ removeErr := unix.Unlinkat(handles[parentIndex], name, unix.AT_REMOVEDIR)
+ if removeErr == nil {
+ continue
+ }
+ if errors.Is(removeErr, unix.ENOENT) || errors.Is(removeErr, unix.ENOTEMPTY) || errors.Is(removeErr, unix.EEXIST) {
+ break
+ }
+ return fmt.Errorf("pruning directory %q: %w", name, removeErr)
+ }
+ return nil
+}
+
+func cleanRelativeParts(relativePath string) ([]string, error) {
+ if relativePath == "" || filepath.IsAbs(relativePath) || filepath.Clean(relativePath) != relativePath {
+ return nil, fmt.Errorf("%w: %q is not a clean relative path", ErrUnsafePath, relativePath)
+ }
+ parts := strings.Split(relativePath, string(filepath.Separator))
+ for _, part := range parts {
+ if part == "" || part == "." || part == ".." {
+ return nil, fmt.Errorf("%w: %q is not a clean relative path", ErrUnsafePath, relativePath)
+ }
+ }
+ return parts, nil
+}
+
+func sameDirectoryEntry(parentFD int, name string, openedFD int) (bool, error) {
+ var opened unix.Stat_t
+ if err := unix.Fstat(openedFD, &opened); err != nil {
+ return false, err
+ }
+ var current unix.Stat_t
+ if err := unix.Fstatat(parentFD, name, ¤t, unix.AT_SYMLINK_NOFOLLOW); err != nil {
+ return false, err
+ }
+ return current.Mode&unix.S_IFMT == unix.S_IFDIR && current.Dev == opened.Dev && current.Ino == opened.Ino, nil
+}
diff --git a/pkg/safefile/remove_unix_test.go b/pkg/safefile/remove_unix_test.go
new file mode 100644
index 000000000..337fb8516
--- /dev/null
+++ b/pkg/safefile/remove_unix_test.go
@@ -0,0 +1,57 @@
+//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris
+
+package safefile
+
+import (
+ "os"
+ "path/filepath"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Race-safe exact removal", func() {
+ DescribeTable("keeps replacement targets untouched after an opened parent is exchanged",
+ func(targetInsideRoot bool) {
+ root := GinkgoT().TempDir()
+ originalRequest := filepath.Join(root, "ephemeral", "request-id")
+ originalCategory := filepath.Join(originalRequest, "audio")
+ Expect(os.MkdirAll(originalCategory, 0750)).To(Succeed())
+ Expect(os.WriteFile(filepath.Join(originalCategory, "input.wav"), []byte("original"), 0640)).To(Succeed())
+
+ replacement := GinkgoT().TempDir()
+ if targetInsideRoot {
+ replacement = filepath.Join(root, "replacement")
+ Expect(os.MkdirAll(replacement, 0750)).To(Succeed())
+ }
+ Expect(os.MkdirAll(filepath.Join(replacement, "audio"), 0750)).To(Succeed())
+ replacementFile := filepath.Join(replacement, "audio", "input.wav")
+ Expect(os.WriteFile(replacementFile, []byte("replacement"), 0640)).To(Succeed())
+
+ renamedRequest := filepath.Join(root, "ephemeral", "opened-request")
+ opened := make(chan struct{})
+ swapped := make(chan struct{})
+ done := make(chan error, 1)
+ go func() {
+ done <- removeExact(root, filepath.Join("ephemeral", "request-id", "audio", "input.wav"), []string{".sha256", ".sha256.target"}, 2, func() {
+ close(opened)
+ <-swapped
+ })
+ }()
+
+ <-opened
+ Expect(os.Rename(originalRequest, renamedRequest)).To(Succeed())
+ Expect(os.Symlink(replacement, originalRequest)).To(Succeed())
+ close(swapped)
+ Expect(<-done).To(Succeed())
+
+ data, err := os.ReadFile(replacementFile)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(data).To(Equal([]byte("replacement")))
+ Expect(originalRequest).To(BeAnExistingFile())
+ Expect(filepath.Join(renamedRequest, "audio", "input.wav")).NotTo(BeAnExistingFile())
+ },
+ Entry("for an internal replacement", true),
+ Entry("for an external replacement", false),
+ )
+})
diff --git a/swagger/docs.go b/swagger/docs.go
index 16fa1de85..487b3ebc6 100644
--- a/swagger/docs.go
+++ b/swagger/docs.go
@@ -5652,6 +5652,12 @@ const docTemplate = `{
"schema.FaceRegisterRequest": {
"type": "object",
"properties": {
+ "embedding": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
"img": {
"type": "string"
},
@@ -5667,6 +5673,10 @@ const docTemplate = `{
"name": {
"type": "string"
},
+ "registered_at": {
+ "description": "original enrollment time when replaying a saved embedding",
+ "type": "string"
+ },
"store": {
"description": "vector store model; empty = local-store default",
"type": "string"
diff --git a/swagger/swagger.json b/swagger/swagger.json
index 2be8aea42..b04e5b4c2 100644
--- a/swagger/swagger.json
+++ b/swagger/swagger.json
@@ -5649,6 +5649,12 @@
"schema.FaceRegisterRequest": {
"type": "object",
"properties": {
+ "embedding": {
+ "type": "array",
+ "items": {
+ "type": "number"
+ }
+ },
"img": {
"type": "string"
},
@@ -5664,6 +5670,10 @@
"name": {
"type": "string"
},
+ "registered_at": {
+ "description": "original enrollment time when replaying a saved embedding",
+ "type": "string"
+ },
"store": {
"description": "vector store model; empty = local-store default",
"type": "string"
diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml
index 58653861b..2dafedd3d 100644
--- a/swagger/swagger.yaml
+++ b/swagger/swagger.yaml
@@ -1138,6 +1138,10 @@ definitions:
type: object
schema.FaceRegisterRequest:
properties:
+ embedding:
+ items:
+ type: number
+ type: array
img:
type: string
labels:
@@ -1148,6 +1152,9 @@ definitions:
type: string
name:
type: string
+ registered_at:
+ description: original enrollment time when replaying a saved embedding
+ type: string
store:
description: vector store model; empty = local-store default
type: string
diff --git a/website/data/stats.yaml b/website/data/stats.yaml
index 8bd8a0f9c..a54e240e8 100644
--- a/website/data/stats.yaml
+++ b/website/data/stats.yaml
@@ -3,10 +3,10 @@
# The four GitHub fields are rewritten by .github/ci/refresh-site-counters.sh,
# which runs weekly from .github/workflows/refresh-site-counters.yml. Editing
# them by hand works but will be overwritten on the next run.
-stars: 48646
-forks: 4377
-contributors: 230
-releases: 136
+stars: 48949
+forks: 4430
+contributors: 237
+releases: 135
# The GitHub API cannot answer for this one, so it is maintained by hand and
# the refresh script carries it through untouched.
|