Merge master into the distributed test branch

Keep the worker control request test and canonicalize its temporary directory.

Assisted-by: Codex:GPT-6

Makefile

@@ -240,7 +240,7 @@ test-ci-scripts:
 ## pure stdlib on purpose so they run without any backend venv; the list is
 ## explicit because their siblings (model_identity_test) import grpc and the
 ## generated protobufs, which only exist inside a built backend.
-PYTHON_HELPER_TESTS?=python_utils_test vllm_utils_test model_utils_test mlx_utils_test parent_watch_test
+PYTHON_HELPER_TESTS?=python_utils_test vllm_utils_test model_utils_test mlx_utils_test parent_watch_test temp_utils_test
 test-python-helpers:
 	cd backend/python/common && python3 -m unittest $(PYTHON_HELPER_TESTS)
 

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=9c6a282337cc83f227cc10428867a478947706ad
+AUDIO_CPP_VERSION?=05e508a70e3600b01454c647cdb122133ba8e64c
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=f62ca29a308724cde5bc99134ede19104b2a3260
+# Upstream pin lives below as DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=f62ca29a308724cde5bc99134ede19104b2a3260
+DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/go/crispasr/gocrispasr.go

@@ -615,10 +615,10 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 		return fmt.Errorf("crispasr: tempfile: %w", err)
 	}
 	dst := tmp.Name()
+	defer func() { _ = os.Remove(dst) }()
 	if err := tmp.Close(); err != nil {
 		return fmt.Errorf("crispasr: close tempfile: %w", err)
 	}
-	defer func() { _ = os.Remove(dst) }()
 
 	if err := writeWAV(dst, pcm, w.sampleRate); err != nil {
 		return err

backend/go/locate-anything-cpp/golocateanythingcpp.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"unsafe"
 
 	"github.com/mudler/LocalAI/pkg/grpc/base"
@@ -109,30 +110,25 @@ func (r *LocateAnythingCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, e
 		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: a text prompt is required (open-vocabulary detection)")
 	}
 
-	// Decode base64 image and write to temp file.
 	imgData, err := base64.StdEncoding.DecodeString(opts.Src)
 	if err != nil {
 		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to decode base64 image: %w", err)
 	}
-
-	tmpFile, err := os.CreateTemp("", "locate-anything-*.img")
-	if err != nil {
-		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to create temp file: %w", err)
-	}
-	defer func() { _ = os.Remove(tmpFile.Name()) }()
-
-	if _, err := tmpFile.Write(imgData); err != nil {
-		_ = tmpFile.Close()
-		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to write temp file: %w", err)
-	}
-	if err := tmpFile.Close(); err != nil {
-		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to close temp file: %w", err)
+	if len(imgData) == 0 {
+		return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: decoded image is empty")
 	}
 
 	// mode 0 = hybrid (Parallel Box Decoding). The JSON return value is unused:
 	// structured detections are read via the accessor functions. Still must
 	// free the returned string.
-	jsonPtr := CapiLocatePath(r.handle, tmpFile.Name(), prompt, 0)
+	jsonPtr := CapiLocateBuffer(
+		r.handle,
+		uintptr(unsafe.Pointer(unsafe.SliceData(imgData))),
+		uintptr(len(imgData)),
+		prompt,
+		0,
+	)
+	runtime.KeepAlive(imgData)
 	if jsonPtr != 0 {
 		CapiFreeString(jsonPtr)
 	}

backend/go/locate-anything-cpp/golocateanythingcpp_unit_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"encoding/base64"
+	"path/filepath"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("LocateAnythingCpp detection input", func() {
+	It("detects from memory when the temporary directory is unavailable", func() {
+		originalLocateBuffer := CapiLocateBuffer
+		originalLocatePath := CapiLocatePath
+		originalGetNDetections := CapiGetNDetections
+		defer func() {
+			CapiLocateBuffer = originalLocateBuffer
+			CapiLocatePath = originalLocatePath
+			CapiGetNDetections = originalGetNDetections
+		}()
+
+		image := []byte("encoded-image")
+		var receivedData uintptr
+		var receivedLength uintptr
+		CapiLocateBuffer = func(_ uintptr, data uintptr, length uintptr, _ string, _ int32) uintptr {
+			receivedData = data
+			receivedLength = length
+			return 0
+		}
+		CapiLocatePath = func(_ uintptr, _ string, _ string, _ int32) uintptr {
+			Fail("path-based detection must not be called")
+			return 0
+		}
+		CapiGetNDetections = func(uintptr) int32 { return 0 }
+		GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "missing"))
+
+		result, err := (&LocateAnythingCpp{handle: 1}).Detect(&pb.DetectOptions{
+			Src:    base64.StdEncoding.EncodeToString(image),
+			Prompt: "the object",
+		})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Detections).To(BeEmpty())
+		Expect(receivedData).NotTo(BeZero())
+		Expect(receivedLength).To(Equal(uintptr(len(image))))
+	})
+
+	It("rejects an empty decoded image", func() {
+		_, err := (&LocateAnythingCpp{handle: 1}).Detect(&pb.DetectOptions{Prompt: "the object"})
+
+		Expect(err).To(MatchError("locate-anything-cpp: decoded image is empty"))
+	})
+})

backend/go/nemo-speech-cpp/Makefile

@@ -12,7 +12,7 @@
 # runs 'make -C backend/go/$(BACKEND) build' and then copies package/), so it
 # has to produce the binary and the package, not just the shared libraries.
 
-NEMO_SPEECH_VERSION?=ffa38cb2408f1e832a36d46fef5e3e1e80d07e6c
+NEMO_SPEECH_VERSION?=a5b6953c4a579a2bbd1c0913ad8a85c2a4d99953
 NEMO_SPEECH_REPO?=https://github.com/NVIDIA/NeMo-Speech.cpp
 
 GOCMD?=go

backend/go/rfdetr-cpp/gorfdetrcpp.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"unsafe"
 
@@ -102,24 +103,12 @@ func (r *RFDetrCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, error) {
 		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: model not loaded")
 	}
 
-	// Decode base64 image and write to temp file.
 	imgData, err := base64.StdEncoding.DecodeString(opts.Src)
 	if err != nil {
 		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to decode base64 image: %w", err)
 	}
-
-	tmpFile, err := os.CreateTemp("", "rfdetr-*.img")
-	if err != nil {
-		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to create temp file: %w", err)
-	}
-	defer func() { _ = os.Remove(tmpFile.Name()) }()
-
-	if _, err := tmpFile.Write(imgData); err != nil {
-		_ = tmpFile.Close()
-		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to write temp file: %w", err)
-	}
-	if err := tmpFile.Close(); err != nil {
-		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to close temp file: %w", err)
+	if len(imgData) == 0 {
+		return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: decoded image is empty")
 	}
 
 	threshold := opts.Threshold
@@ -127,10 +116,18 @@ func (r *RFDetrCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, error) {
 		threshold = 0.5
 	}
 
-	// JSON output from detect_path is unused: we read structured detections via
+	// JSON output from the detection ABI is unused: we read structured detections via
 	// the accessor functions. Still must free the returned string.
 	var jsonPtr uintptr
-	rc := CapiDetectPath(r.handle, tmpFile.Name(), threshold, uint32(defaultTopK), &jsonPtr)
+	rc := CapiDetectBuffer(
+		r.handle,
+		uintptr(unsafe.Pointer(unsafe.SliceData(imgData))),
+		uintptr(len(imgData)),
+		threshold,
+		uint32(defaultTopK),
+		&jsonPtr,
+	)
+	runtime.KeepAlive(imgData)
 	if jsonPtr != 0 {
 		CapiFreeString(jsonPtr)
 	}

backend/go/rfdetr-cpp/gorfdetrcpp_unit_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"encoding/base64"
+	"path/filepath"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("RFDetrCpp detection input", func() {
+	It("detects from memory when the temporary directory is unavailable", func() {
+		originalDetectBuffer := CapiDetectBuffer
+		originalDetectPath := CapiDetectPath
+		originalFreeString := CapiFreeString
+		originalGetNDetections := CapiGetNDetections
+		defer func() {
+			CapiDetectBuffer = originalDetectBuffer
+			CapiDetectPath = originalDetectPath
+			CapiFreeString = originalFreeString
+			CapiGetNDetections = originalGetNDetections
+		}()
+
+		image := []byte("encoded-image")
+		var receivedData uintptr
+		var receivedLength uintptr
+		CapiDetectBuffer = func(_ uintptr, data uintptr, length uintptr, _ float32, _ uint32, _ *uintptr) int32 {
+			receivedData = data
+			receivedLength = length
+			return 0
+		}
+		CapiDetectPath = func(_ uintptr, _ string, _ float32, _ uint32, _ *uintptr) int32 {
+			Fail("path-based detection must not be called")
+			return -1
+		}
+		CapiFreeString = func(uintptr) {}
+		CapiGetNDetections = func(uintptr) int32 { return 0 }
+		GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "missing"))
+
+		result, err := (&RFDetrCpp{handle: 1}).Detect(&pb.DetectOptions{
+			Src: base64.StdEncoding.EncodeToString(image),
+		})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Detections).To(BeEmpty())
+		Expect(receivedData).NotTo(BeZero())
+		Expect(receivedLength).To(Equal(uintptr(len(image))))
+	})
+
+	It("rejects an empty decoded image", func() {
+		_, err := (&RFDetrCpp{handle: 1}).Detect(&pb.DetectOptions{})
+
+		Expect(err).To(MatchError("rfdetr-cpp: decoded image is empty"))
+	})
+})

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -1144,17 +1144,25 @@ static uint8_t* load_and_resize_image(const char* path, int target_width, int ta
 // Write sd.cpp's audio buffer to a temp WAV file (IEEE float, interleaved).
 // sd_audio_t.data is planar (all channel 0 samples, then channel 1, etc.) — we
 // interleave on the fly so ffmpeg's standard wav demuxer can read it directly.
-// Returns 0 on success and fills wav_path (must be at least 64 bytes).
+// Returns 0 on success and fills wav_path.
 static int write_planar_float_wav(const sd_audio_t* a, char* wav_path, size_t wav_path_sz) {
     if (!a || !a->data || a->sample_count == 0 || a->channels == 0 || a->sample_rate == 0) {
         return -1;
     }
 
-    snprintf(wav_path, wav_path_sz, "/tmp/gosd-audio-XXXXXX.wav");
+    const char* temp_dir = getenv("TMPDIR");
+    if (!temp_dir || temp_dir[0] == '\0') {
+        temp_dir = "/tmp";
+    }
+    int path_len = snprintf(wav_path, wav_path_sz, "%s/gosd-audio-XXXXXX.wav", temp_dir);
+    if (path_len < 0 || (size_t)path_len >= wav_path_sz) {
+        fprintf(stderr, "temporary directory path is too long\n");
+        return -1;
+    }
     int fd = mkstemps(wav_path, 4);
     if (fd < 0) { perror("mkstemps wav"); return -1; }
     FILE* f = fdopen(fd, "wb");
-    if (!f) { perror("fdopen wav"); close(fd); return -1; }
+    if (!f) { perror("fdopen wav"); close(fd); unlink(wav_path); return -1; }
 
     uint64_t frames = a->sample_count;
     uint32_t channels = a->channels;
@@ -1221,7 +1229,7 @@ static int ffmpeg_mux_raw_to_mp4(sd_image_t* frames, int num_frames, int fps,
     snprintf(fps_str, sizeof(fps_str), "%d", fps);
 
     // Optional audio: write a temp WAV file if the model produced audio.
-    char wav_path[64] = {0};
+    char wav_path[4096] = {0};
     bool have_audio = false;
     if (audio && audio->data && audio->sample_count > 0 && audio->channels > 0 && audio->sample_rate > 0) {
         if (write_planar_float_wav(audio, wav_path, sizeof(wav_path)) == 0) {

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=52a939a2a762224e255d366c1182b2af4dd1a032
+WHISPER_CPP_VERSION?=c44b60b8053bbf2a5c1e014f11323fb3f2485177
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/python/chatterbox/backend.py

@@ -19,6 +19,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
+from temp_utils import cleanup_paths
 
 import tempfile
 
@@ -115,11 +116,6 @@ def merge_audio_files(audio_files, output_path, sample_rate):
     # Save the merged audio
     ta.save(output_path, merged_waveform, sample_rate)
     
-    # Clean up temporary files
-    for audio_file in audio_files:
-        if os.path.exists(audio_file):
-            os.remove(audio_file)
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
@@ -226,19 +222,20 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 text_chunks = split_text_at_word_boundary(request.text, max_length=250)
                 print(f"Splitting text into chunks of 250 characters: {len(text_chunks)}", file=sys.stderr)
                 # Generate audio for each chunk
-                temp_audio_files = []
-                for i, chunk in enumerate(text_chunks):
-                    # Generate audio for this chunk
-                    wav = self.model.generate(chunk, **kwargs)
-                    
-                    # Create temporary file for this chunk
-                    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
-                    temp_file.close()
-                    ta.save(temp_file.name, wav, self.model.sr)
-                    temp_audio_files.append(temp_file.name)
-                
-                # Merge all audio files
-                merge_audio_files(temp_audio_files, request.dst, self.model.sr)
+                with cleanup_paths() as temp_audio_files:
+                    for i, chunk in enumerate(text_chunks):
+                        # Generate audio for this chunk
+                        wav = self.model.generate(chunk, **kwargs)
+
+                        # Register ownership before saving so a partial write is
+                        # removed too when generation or encoding fails.
+                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
+                        temp_file.close()
+                        temp_audio_files.append(temp_file.name)
+                        ta.save(temp_file.name, wav, self.model.sr)
+
+                    # Merge all audio files
+                    merge_audio_files(temp_audio_files, request.dst, self.model.sr)
             else:
                 # Generate audio using ChatterboxTTS for short text
                 wav = self.model.generate(request.text, **kwargs)

backend/python/common/temp_utils.py

@@ -0,0 +1,36 @@
+import base64
+import contextlib
+import os
+import tempfile
+
+
+@contextlib.contextmanager
+def materialize_base64(data, suffix=""):
+    """Materialize base64 data for a path-only library and always remove it."""
+    descriptor, path = tempfile.mkstemp(prefix="localai-media-", suffix=suffix)
+    try:
+        with os.fdopen(descriptor, "wb") as output:
+            descriptor = None
+            output.write(base64.b64decode(data))
+        yield path
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+
+
+@contextlib.contextmanager
+def cleanup_paths():
+    """Collect temporary paths and remove them on success or failure."""
+    paths = []
+    try:
+        yield paths
+    finally:
+        for path in paths:
+            try:
+                os.remove(path)
+            except OSError:
+                pass

backend/python/common/temp_utils_test.py

@@ -0,0 +1,41 @@
+import os
+import tempfile
+import unittest
+from unittest import mock
+
+from temp_utils import cleanup_paths, materialize_base64
+
+
+class MaterializeBase64Test(unittest.TestCase):
+    def test_removes_materialized_file_after_success(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with mock.patch.object(tempfile, "tempdir", directory):
+                with materialize_base64("aGVsbG8=", suffix=".data") as path:
+                    with open(path, "rb") as materialized:
+                        self.assertEqual(materialized.read(), b"hello")
+                self.assertFalse(os.path.exists(path))
+
+    def test_removes_materialized_file_when_consumer_fails(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with mock.patch.object(tempfile, "tempdir", directory):
+                with self.assertRaisesRegex(RuntimeError, "decode failed"):
+                    with materialize_base64("aGVsbG8="):
+                        raise RuntimeError("decode failed")
+                self.assertEqual(os.listdir(directory), [])
+
+
+class CleanupPathsTest(unittest.TestCase):
+    def test_removes_every_registered_path_after_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            paths = [os.path.join(directory, name) for name in ("one.wav", "two.wav")]
+            with self.assertRaisesRegex(RuntimeError, "merge failed"):
+                with cleanup_paths() as registered:
+                    for path in paths:
+                        open(path, "wb").close()
+                        registered.append(path)
+                    raise RuntimeError("merge failed")
+            self.assertEqual(os.listdir(directory), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/longcat-video/backend.py

@@ -6,6 +6,7 @@ import datetime
 import gc
 import math
 import os
+import shutil
 import signal
 import subprocess
 import sys
@@ -888,6 +889,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
     def _release_model(self):
         self.pipeline = None
         self.model_kind = None
+        try:
+            if hasattr(self, "dist") and self.dist.is_initialized():
+                self.dist.destroy_process_group()
+        finally:
+            if self._dist_store_dir is not None:
+                shutil.rmtree(self._dist_store_dir, ignore_errors=True)
+                self._dist_store_dir = None
         gc.collect()
         if hasattr(self, "torch") and self.torch.cuda.is_available():
             self.torch.cuda.empty_cache()

backend/python/vllm-omni/backend.py

@@ -19,7 +19,6 @@ import base64
 import io
 import json
 import gc
-import tempfile
 
 from PIL import Image
 import torch
@@ -34,6 +33,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from temp_utils import materialize_base64
 from vllm_utils import parse_options, messages_to_dicts, setup_parsers
 
 
@@ -118,13 +118,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return video_to_ndarrays(video_path, num_frames=16)
         # Try base64 decode
         try:
-            timestamp = str(int(time.time() * 1000))
-            p = os.path.join(tempfile.gettempdir(), f"vl-{timestamp}.data")
-            with open(p, "wb") as f:
-                f.write(base64.b64decode(video_path))
-            video = VideoAsset(name=p).np_ndarrays
-            os.remove(p)
-            return video
+            with materialize_base64(video_path, suffix=".data") as path:
+                return VideoAsset(name=path).np_ndarrays
         except:
             return None
 
@@ -136,15 +131,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return (audio_signal.astype(np.float32), sr)
         # Try base64 decode
         try:
-            audio_data = base64.b64decode(audio_path)
-            # Save to temp file and load
-            timestamp = str(int(time.time() * 1000))
-            p = os.path.join(tempfile.gettempdir(), f"audio-{timestamp}.wav")
-            with open(p, "wb") as f:
-                f.write(audio_data)
-            audio_signal, sr = librosa.load(p, sr=16000)
-            os.remove(p)
-            return (audio_signal.astype(np.float32), sr)
+            with materialize_base64(audio_path, suffix=".wav") as path:
+                audio_signal, sr = librosa.load(path, sr=16000)
+                return (audio_signal.astype(np.float32), sr)
         except:
             return None
 

backend/python/vllm/backend.py

@@ -10,7 +10,6 @@ import os
 import json
 import time
 import gc
-import tempfile
 from typing import List
 from PIL import Image
 
@@ -23,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from python_utils import attach_media_parts
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from temp_utils import materialize_base64
 from vllm_utils import apply_options_to_engine_args, normalize_option_key
 
 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -1005,13 +1005,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             Video: The loaded video.
         """
         try:
-            timestamp = str(int(time.time() * 1000))  # Generate timestamp
-            p = os.path.join(tempfile.gettempdir(), f"vl-{timestamp}.data")
-            with open(p, "wb") as f:
-                f.write(base64.b64decode(video_path))
-            video = VideoAsset(name=p).np_ndarrays
-            os.remove(p)
-            return video
+            with materialize_base64(video_path, suffix=".data") as path:
+                return VideoAsset(name=path).np_ndarrays
         except Exception as e:
             print(f"Error loading video {video_path}: {e}", file=sys.stderr)
             return None

core/http/endpoints/openai/realtime.go

@@ -12,6 +12,7 @@ import (
 	"math"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -633,6 +634,17 @@ func runRealtimeSession(application *application.Application, t Transport, model
 		sendError(t, "model_load_error", "Failed to load model", "", "")
 		return
 	}
+	if wrapped, ok := m.(*wrappedModel); ok {
+		resolvedVoice, params, release, resolveErr := resolveRealtimeVoice(context.Background(), session.Voice, wrapped.TTSConfig, application.VoiceProfileStore())
+		if resolveErr != nil {
+			xlog.Error("failed to resolve realtime voice", "error", resolveErr)
+			sendError(t, "voice_profile_error", resolveErr.Error(), "", "")
+			return
+		}
+		defer release()
+		session.Voice = resolvedVoice
+		wrapped.ttsParams = params
+	}
 	session.ModelInterface = m
 	// A pipeline-seeded option list gets its scoring prompt prewarmed
 	// alongside the model warm-up below, so the session's first turn
@@ -1923,7 +1935,7 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt
 	// Generate an LLM response only when there is a transcript to feed it. A
 	// sound-detection-only session (no transcription) has no LLM stage, so it
 	// stops here after emitting the sound-detection event.
-	if session.InputAudioTranscription != nil && !session.TranscriptionOnly {
+	if session.InputAudioTranscription != nil && !session.TranscriptionOnly && strings.TrimSpace(transcript) != "" {
 		generateResponse(ctx, session, utt, transcript, speaker, conv, t)
 	}
 }

core/http/endpoints/openai/realtime_model.go

@@ -6,7 +6,9 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"maps"
 	"strings"
 	"sync"
 	"time"
@@ -18,6 +20,7 @@ import (
 	"github.com/mudler/LocalAI/core/http/middleware"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/services/routing/router"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
 	"github.com/mudler/LocalAI/core/templates"
 	"github.com/mudler/LocalAI/pkg/functions"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -35,6 +38,7 @@ var (
 // which are for Any-To-Any models, but instead we will call a pipeline (for e.g STT->LLM->TTS)
 type wrappedModel struct {
 	TTSConfig            *config.ModelConfig
+	ttsParams            map[string]string
 	TranscriptionConfig  *config.ModelConfig
 	LLMConfig            *config.ModelConfig
 	VADConfig            *config.ModelConfig
@@ -391,11 +395,35 @@ func newRealtimeDecisionID() string {
 }
 
 func (m *wrappedModel) TTS(ctx context.Context, text, voice, language string) (string, *proto.Result, error) {
-	return backend.ModelTTS(ctx, text, voice, language, "", nil, m.modelLoader, m.appConfig, *m.TTSConfig)
+	return backend.ModelTTS(ctx, text, voice, language, "", maps.Clone(m.ttsParams), m.modelLoader, m.appConfig, *m.TTSConfig)
 }
 
 func (m *wrappedModel) TTSStream(ctx context.Context, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error {
-	return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, onAudio)
+	return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, maps.Clone(m.ttsParams), onAudio)
+}
+
+func resolveRealtimeVoice(ctx context.Context, configuredVoice string, ttsConfig *config.ModelConfig, profiles *voiceprofile.Store) (string, map[string]string, func(), error) {
+	if !voiceprofile.IsReference(configuredVoice) {
+		return configuredVoice, nil, func() {}, nil
+	}
+	profileID, valid := voiceprofile.ParseReference(configuredVoice)
+	if !valid {
+		return "", nil, nil, fmt.Errorf("invalid voice profile reference %q", configuredVoice)
+	}
+	if config.VoiceCloningForModel(ttsConfig) == nil {
+		return "", nil, nil, fmt.Errorf("selected TTS model does not support reference-audio voice cloning")
+	}
+	if profiles == nil {
+		return "", nil, nil, fmt.Errorf("voice profile store is unavailable")
+	}
+	profile, referencePath, release, err := profiles.LeaseAudio(ctx, profileID)
+	if err != nil {
+		if errors.Is(err, voiceprofile.ErrNotFound) {
+			return "", nil, nil, fmt.Errorf("voice profile not found: %w", err)
+		}
+		return "", nil, nil, fmt.Errorf("resolve voice profile: %w", err)
+	}
+	return referencePath, map[string]string{"ref_text": profile.Transcript}, release, nil
 }
 
 func (m *wrappedModel) TranscribeStream(ctx context.Context, audio, language string, translate, diarize bool, prompt string, onDelta func(text string)) (*schema.TranscriptionResult, error) {
@@ -674,11 +702,11 @@ const wavStreamHeaderBytes = 44
 // callback, which wants raw PCM plus the sample rate. The header is buffered
 // until complete, the sample rate is read from it, and subsequent bytes are
 // forwarded as PCM.
-func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error {
+func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, params map[string]string, onAudio func(pcm []byte, sampleRate int) error) error {
 	var header []byte
 	headerDone := false
 	sampleRate := 0
-	return backend.ModelTTSStream(ctx, text, voice, language, "", nil, ml, appConfig, ttsConfig, func(b []byte) error {
+	return backend.ModelTTSStream(ctx, text, voice, language, "", params, ml, appConfig, ttsConfig, func(b []byte) error {
 		if headerDone {
 			if len(b) == 0 {
 				return nil

core/http/endpoints/openai/realtime_semantic_vad_test.go

@@ -355,6 +355,19 @@ var _ = Describe("commitUtteranceWithTranscript", func() {
 
 		Expect(tr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1))
 	})
+
+	It("does not generate a response for a blank transcript", func() {
+		session, model := itSession(nil)
+		model.transcribeFinal = &schema.TranscriptionResult{Text: " \t\n"}
+		tr := &fakeTransport{}
+		conv := &Conversation{}
+
+		commitUtterance(context.Background(), []byte{1, 2}, session, conv, tr)
+
+		Expect(tr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1))
+		Expect(conv.Items).To(BeEmpty())
+		Expect(tr.countEvents(types.ServerEventTypeResponseCreated)).To(Equal(0))
+	})
 })
 
 // transcribeUtterance is the retranscribe gate's offline decode of the

core/http/endpoints/openai/realtime_voice_profile_test.go

@@ -0,0 +1,180 @@
+package openai
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"time"
+
+	grpcPkg "github.com/mudler/LocalAI/pkg/grpc"
+	"github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	"google.golang.org/grpc"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func realtimeProfileWAV(duration time.Duration) []byte {
+	const (
+		sampleRate    = 16000
+		channels      = 1
+		bitsPerSample = 16
+	)
+	dataSize := int(duration.Seconds() * sampleRate * channels * bitsPerSample / 8)
+	buf := bytes.NewBuffer(nil)
+	buf.WriteString("RIFF")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
+	buf.WriteString("WAVEfmt ")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(16))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(1))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(channels))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*channels*bitsPerSample/8))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(channels*bitsPerSample/8))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample))
+	buf.WriteString("data")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
+	buf.Write(make([]byte, dataSize))
+	return buf.Bytes()
+}
+
+var _ = Describe("realtime pipeline voice profiles", func() {
+	It("resolves a saved profile to an immutable lease and transcript", func(ctx SpecContext) {
+		store := voiceprofile.NewStore(GinkgoT().TempDir())
+		DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
+		profile, err := store.Create(ctx, voiceprofile.CreateInput{
+			Name:             "Narrator",
+			Language:         "en-US",
+			Transcript:       "The reference transcript.",
+			ConsentConfirmed: true,
+		}, bytes.NewReader(realtimeProfileWAV(time.Second)))
+		Expect(err).NotTo(HaveOccurred())
+
+		voice, params, release, err := resolveRealtimeVoice(ctx, profile.Voice, &config.ModelConfig{
+			Name:      "clone-base",
+			Backend:   "qwen3-tts-cpp",
+			TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)},
+		}, store)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(voice).To(BeAnExistingFile())
+		Expect(params).To(Equal(map[string]string{"ref_text": "The reference transcript."}))
+		release()
+		release()
+		Expect(voice).NotTo(BeAnExistingFile())
+	})
+
+	It("leaves an ordinary backend voice unchanged with no parameters", func() {
+		voice, params, release, err := resolveRealtimeVoice(context.Background(), "speaker-7", &config.ModelConfig{}, nil)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(voice).To(Equal("speaker-7"))
+		Expect(params).To(BeNil())
+		Expect(release).NotTo(BeNil())
+		Expect(func() { release(); release() }).NotTo(Panic())
+	})
+
+	DescribeTable("returns actionable reference errors",
+		func(configuredVoice string, cfg *config.ModelConfig, store *voiceprofile.Store, expected string) {
+			_, _, release, err := resolveRealtimeVoice(context.Background(), configuredVoice, cfg, store)
+			Expect(err).To(MatchError(ContainSubstring(expected)))
+			Expect(release).To(BeNil())
+		},
+		Entry("malformed reference", "localai://voice-profiles/not-a-uuid", &config.ModelConfig{}, nil, "invalid voice profile reference"),
+		Entry("unsupported model", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Backend: "piper"}, nil, "does not support reference-audio voice cloning"),
+		Entry("unavailable store", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)}}, nil, "voice profile store is unavailable"),
+	)
+
+	It("reports a missing profile", func() {
+		store := voiceprofile.NewStore(GinkgoT().TempDir())
+		DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
+		_, _, release, err := resolveRealtimeVoice(context.Background(), "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{
+			Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)},
+		}, store)
+		Expect(errors.Is(err, voiceprofile.ErrNotFound)).To(BeTrue())
+		Expect(err.Error()).To(ContainSubstring("voice profile not found"))
+		Expect(release).To(BeNil())
+	})
+})
+
+type recordingTTSBackend struct {
+	grpcPkg.Backend
+	requests []*proto.TTSRequest
+}
+
+func (b *recordingTTSBackend) HealthCheck(context.Context) (bool, error) { return true, nil }
+func (b *recordingTTSBackend) IsBusy() bool                              { return false }
+
+func (b *recordingTTSBackend) record(req *proto.TTSRequest) {
+	b.requests = append(b.requests, req)
+	req.Params["ref_text"] = "backend mutation"
+}
+
+func (b *recordingTTSBackend) TTS(_ context.Context, req *proto.TTSRequest, _ ...grpc.CallOption) (*proto.Result, error) {
+	b.record(req)
+	return &proto.Result{Success: true}, nil
+}
+
+func (b *recordingTTSBackend) TTSStream(_ context.Context, req *proto.TTSRequest, callback func(*proto.Reply), _ ...grpc.CallOption) error {
+	b.record(req)
+	header := make([]byte, wavStreamHeaderBytes)
+	binary.LittleEndian.PutUint32(header[24:28], 24000)
+	callback(&proto.Reply{Audio: header})
+	return nil
+}
+
+var _ = Describe("wrappedModel voice profile parameters", func() {
+	var (
+		wrapped         *wrappedModel
+		backendRecorder *recordingTTSBackend
+	)
+
+	BeforeEach(func() {
+		state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
+		Expect(err).NotTo(HaveOccurred())
+		appConfig := config.NewApplicationConfig(config.WithSystemState(state))
+		appConfig.GeneratedContentDir = GinkgoT().TempDir()
+		loader := model.NewModelLoader(state)
+		backendRecorder = &recordingTTSBackend{}
+		cfg := &config.ModelConfig{Name: "tts-test", Backend: "test"}
+		cfg.Model = "weights"
+		loaded := model.NewModelWithClient(cfg.ModelID(), "in-process", backendRecorder)
+		loaded.MarkHealthy()
+		_, err = loader.LoadModel(cfg.ModelID(), cfg.Model, func(_, _, _ string) (*model.Model, error) { return loaded, nil })
+		Expect(err).NotTo(HaveOccurred())
+		wrapped = &wrappedModel{
+			TTSConfig:   cfg,
+			ttsParams:   map[string]string{"ref_text": "Original transcript"},
+			modelLoader: loader,
+			appConfig:   appConfig,
+		}
+	})
+
+	It("forwards a fresh transcript parameter map to every unary request", func() {
+		_, _, err := wrapped.TTS(context.Background(), "one", "voice.wav", "en")
+		Expect(err).NotTo(HaveOccurred())
+		_, _, err = wrapped.TTS(context.Background(), "two", "voice.wav", "en")
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(backendRecorder.requests).To(HaveLen(2))
+		Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(backendRecorder.requests[1].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript"))
+	})
+
+	It("forwards a copied transcript parameter map to streaming requests", func() {
+		err := wrapped.TTSStream(context.Background(), "one", "voice.wav", "en", func([]byte, int) error { return nil })
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(backendRecorder.requests).To(HaveLen(1))
+		Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript"))
+	})
+})
+
+func ptrTo[T any](value T) *T { return &value }

core/services/worker/ephemeral_capacity_test.go

@@ -43,7 +43,7 @@ func (capacityShortWriter) Write(p []byte) (int, error) {
 
 var _ = Describe("EphemeralCapacityGuard", func() {
 	It("derives bounded defaults and preserves positive overrides", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		limit, headroom, err := effectiveEphemeralCapacity([]string{root}, 0, -1)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(limit).To(BeNumerically(">", 0))
@@ -57,8 +57,8 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("accounts existing regular files without following symlinks", func() {
-		root := GinkgoT().TempDir()
-		outside := filepath.Join(GinkgoT().TempDir(), "outside.bin")
+		root := canonicalWorkerTempDir()
+		outside := filepath.Join(canonicalWorkerTempDir(), "outside.bin")
 		Expect(os.WriteFile(filepath.Join(root, "existing.bin"), make([]byte, 6), 0o600)).To(Succeed())
 		Expect(os.WriteFile(outside, make([]byte, 100), 0o600)).To(Succeed())
 		Expect(os.Symlink(outside, filepath.Join(root, "outside-link"))).To(Succeed())
@@ -77,7 +77,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("serializes competing reservations", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 1, 0)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -109,7 +109,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("makes only an equal active reservation idempotent", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "nested", "payload.bin")
@@ -128,7 +128,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("retains committed bytes when the same path starts another reservation", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -147,7 +147,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("retains startup-accounted bytes when the path is reserved", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		path := filepath.Join(root, "payload.bin")
 		Expect(os.WriteFile(path, make([]byte, 4), 0o600)).To(Succeed())
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
@@ -163,7 +163,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("commits the regular file's actual size", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -176,7 +176,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("preserves configured filesystem headroom", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 1<<30, 1<<62)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -189,7 +189,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("reserves bounded chunks before forwarding unknown-length input", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, ephemeralCapacityWriteChunk+1, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -210,7 +210,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("waits for an open bounded writer before committing", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -254,7 +254,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("does not share pending capacity between concurrent writers", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -290,7 +290,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rolls back bytes the destination writer does not accept", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 5, 0)
 		Expect(err).NotTo(HaveOccurred())
 		writer, err := guard.NewWriter(filepath.Join(root, "payload.bin"), capacityShortWriter{})
@@ -304,8 +304,8 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects paths outside roots and through symlinks", func() {
-		root := GinkgoT().TempDir()
-		outside := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
+		outside := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 100, 0)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -319,7 +319,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("supports recovery tree accounting without dropping active reservations", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		active := filepath.Join(root, "active", "payload.bin")
@@ -335,7 +335,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("waits for pre-release reservations before request cleanup scans", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-1", "input.wav")
@@ -357,7 +357,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects staging after request cleanup begins", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(guard.BeginRequestRelease(context.Background(), "request-1")).To(Succeed())
@@ -370,7 +370,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("leaves a late commit recoverable when release times out", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-1", "late.wav")
@@ -386,7 +386,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("bounds release markers without reopening registered work", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(guard.BeginRequestOperation("request-pinned")).To(Succeed())
@@ -411,7 +411,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("applies backpressure at the release-pin cap and clears ownership", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
@@ -438,7 +438,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("makes committed files recoverable when pin backpressure expires", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
@@ -459,7 +459,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects a registered cache-hit claim after pin backpressure expires", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
 		Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
 		Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())

core/services/worker/ephemeral_cleanup_test.go

@@ -24,7 +24,7 @@ var _ = Describe("Worker ephemeral staging cleanup", func() {
 		return dir
 	}
 
-	BeforeEach(func() { stagingDir = GinkgoT().TempDir() })
+	BeforeEach(func() { stagingDir = canonicalWorkerTempDir() })
 
 	It("removes staged request directories older than the TTL", func() {
 		old := mkEphemeral("aaaa1111", 48*time.Hour)
@@ -57,7 +57,7 @@ var _ = Describe("Worker ephemeral staging cleanup", func() {
 	})
 
 	It("sweeps both transport roots by newest descendant and skips active requests", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		httpRoot := filepath.Join(stagingDir, "ephemeral")
 		s3Root := filepath.Join(cacheDir, "ephemeral")
 		guard, err := NewEphemeralCapacityGuard([]string{httpRoot, s3Root}, 8, 0)

core/services/worker/file_staging_release_test.go

@@ -108,7 +108,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("protects a startup-accounted HTTP cache hit through authenticated repeated probes", func() {
-		stagingDir := GinkgoT().TempDir()
+		stagingDir := canonicalWorkerTempDir()
 		root := filepath.Join(stagingDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		remotePath := filepath.Join(stagingDir, filepath.FromSlash(key))
@@ -128,11 +128,11 @@ var _ = Describe("Worker exact-key staging release", func() {
 		Expect(err).NotTo(HaveOccurred())
 		addr := listener.Addr().String()
 		Expect(listener.Close()).To(Succeed())
-		server, err := nodes.StartFileTransferServerWithCapacity(addr, stagingDir, GinkgoT().TempDir(), GinkgoT().TempDir(), "secret", 0, nil, guard)
+		server, err := nodes.StartFileTransferServerWithCapacity(addr, stagingDir, canonicalWorkerTempDir(), canonicalWorkerTempDir(), "secret", 0, nil, guard)
 		Expect(err).NotTo(HaveOccurred())
 		DeferCleanup(nodes.ShutdownFileTransferServer, server)
 
-		localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+		localPath := filepath.Join(canonicalWorkerTempDir(), "input.wav")
 		Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
 		stager := nodes.NewHTTPFileStager(func(string) (string, error) { return addr, nil }, "secret", func(string) func(context.Context, string, string) (net.Conn, error) {
 			return (&net.Dialer{}).DialContext
@@ -152,7 +152,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("claims a startup-scanned cache hit against stale recovery until release", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -181,7 +181,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("downloads again when a cache file disappears while being claimed", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
 		Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
@@ -199,7 +199,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("makes repeated cache-hit claims idempotent", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -222,7 +222,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("capacity-checks growth of a startup-scanned cache file", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -243,7 +243,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("reserves S3 object size before download and releases it with the exact key", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		store := &stagingObjectStore{payload: []byte("data")}
 		fm, err := storage.NewFileManager(store, cacheDir)
@@ -263,7 +263,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rejects an oversized S3 object before starting its download", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		store := &stagingObjectStore{payload: []byte("oversized")}
 		fm, err := storage.NewFileManager(store, cacheDir)
 		Expect(err).NotTo(HaveOccurred())
@@ -276,7 +276,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rolls back an S3 reservation when the download fails", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		store := &stagingObjectStore{payload: []byte("data"), getErr: errors.New("download failed")}
 		fm, err := storage.NewFileManager(store, cacheDir)
@@ -290,7 +290,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("removes only the exact cache file and upload sidecars", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 		Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 		target := filepath.Join(categoryDir, "input.wav")
@@ -308,7 +308,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("succeeds for a missing file and prunes empty category and request directories", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 		Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 
@@ -321,8 +321,8 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rejects traversal and symlink escapes", func() {
-		cacheDir := GinkgoT().TempDir()
-		outsideDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
+		outsideDir := canonicalWorkerTempDir()
 		outsidePath := filepath.Join(outsideDir, "input.wav")
 		Expect(os.WriteFile(outsidePath, []byte("keep"), 0640)).To(Succeed())
 		requestDir := filepath.Join(cacheDir, "ephemeral", "request-id")
@@ -342,7 +342,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 
 	It("rejects symlinked files and sidecars without deleting their targets", func() {
 		for _, linkedName := range []string{"input.wav", "input.wav.sha256", "input.wav.sha256.target"} {
-			cacheDir := GinkgoT().TempDir()
+			cacheDir := canonicalWorkerTempDir()
 			categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 			Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 			target := filepath.Join(categoryDir, "input.wav")
@@ -359,7 +359,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("registers an exact release handler", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		path := filepath.Join(cacheDir, "ephemeral", "request-id", "audio", "input.wav")
 		Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
 		Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
@@ -381,7 +381,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("releases a request batch through one worker control request", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		keys := []string{
 			"ephemeral/audio/request-id/input.wav",
 			"ephemeral/images/request-id/frame.jpg",
@@ -411,7 +411,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("returns validation errors through the release handler", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		fm, err := storage.NewFileManager(nil, cacheDir)
 		Expect(err).NotTo(HaveOccurred())
 		mux := http.NewServeMux()

core/services/worker/supervisor.go

@@ -732,6 +732,7 @@ func (s *backendSupervisor) reapDeadProcess(key string, bp *backendProcess) {
 	if bp == nil {
 		return
 	}
+	s.cleanupProcessRuntime(bp.proc)
 	if bp.port <= 0 {
 		xlog.Error("Cannot recycle backend port: dead process has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
 		return
@@ -749,6 +750,7 @@ func (s *backendSupervisor) releaseBackendStart(key string, bp *backendProcess)
 		return
 	}
 	delete(s.processes, key)
+	s.cleanupProcessRuntime(bp.proc)
 	if bp.port <= 0 {
 		xlog.Error("Cannot recycle backend port: startup has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
 		return
@@ -1090,6 +1092,7 @@ func (s *backendSupervisor) finishBackendStop(key string, bp *backendProcess, st
 		return fmt.Errorf("stopping backend process %s: %w", key, stopErr)
 	}
 	delete(s.processes, key)
+	s.cleanupProcessRuntime(bp.proc)
 	if bp.port <= 0 {
 		xlog.Error("Cannot recycle backend port: process has invalid recorded port", "backend", key, "addr", bp.addr, "port", bp.port)
 		return nil
@@ -1098,6 +1101,14 @@ func (s *backendSupervisor) finishBackendStop(key string, bp *backendProcess, st
 	return nil
 }
 
+func (s *backendSupervisor) cleanupProcessRuntime(proc *process.Process) {
+	// Some focused supervisor tests provide synthetic process handles without a
+	// ModelLoader. Production processes always come from s.ml.StartProcess.
+	if s.ml != nil {
+		s.ml.CleanupProcessRuntime(proc)
+	}
+}
+
 // stopAllBackends stops all running backend processes.
 func (s *backendSupervisor) stopAllBackends(force bool) {
 	s.mu.Lock()

core/services/worker/worker_suite_test.go

@@ -1,6 +1,7 @@
 package worker
 
 import (
+	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -11,3 +12,11 @@ func TestWorker(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Worker Suite")
 }
+
+// Capacity guards reject symlink components, including macOS /var -> /private/var.
+func canonicalWorkerTempDir() string {
+	GinkgoHelper()
+	dir, err := filepath.EvalSymlinks(GinkgoT().TempDir())
+	Expect(err).NotTo(HaveOccurred())
+	return dir
+}

docs/content/features/text-to-audio.md

@@ -124,6 +124,25 @@ Reference selection follows this order:
 
 When a saved profile is selected, LocalAI supplies both its private WAV and exact transcript for that request. It does not rewrite the model YAML or copy the recording into the model directory.
 
+### Realtime pipeline default
+
+Set `tts.voice` on a realtime pipeline model to use a saved Voice Library profile as the session default:
+
+```yaml
+name: gpt-realtime
+tts:
+  voice: localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000
+pipeline:
+  vad: silero-vad-ggml
+  transcription: whisper-large-turbo
+  llm: qwen3-4b
+  tts: qwen3-tts-base
+```
+
+LocalAI resolves this profile when the realtime session starts. The selected TTS model must support Voice Library cloning.
+
+This feature does not resolve Voice Library URIs sent later through realtime `session.update`. You can still use `session.update` with ordinary backend voice names or IDs.
+
 #### Supported backend and model variants
 
 | Backend | Automatically compatible variants |

docs/content/reference/cli-reference.md

@@ -27,9 +27,16 @@ Complete reference for all LocalAI command-line interface (CLI) parameters and e
 | `--upload-path` | `TMPDIR/localai-UID/upload` | Path to store uploads from files API. Defaults under the OS temp dir (`$TMPDIR`, falling back to `/tmp`), scoped to the current user's UID. | `$LOCALAI_UPLOAD_PATH`, `$UPLOAD_PATH` |
 | `--localai-config-dir` | `BASEPATH/configuration` | Directory for dynamic loading of certain configuration files (currently runtime_settings.json, api_keys.json, and external_backends.json). See [Runtime Settings]({{%relref "features/runtime-settings" %}}) for web-based configuration. | `$LOCALAI_CONFIG_DIR` |
 | `--localai-config-dir-poll-interval` | | Time duration to poll the LocalAI Config Dir if your system has broken fsnotify events (example: `1m`) | `$LOCALAI_CONFIG_DIR_POLL_INTERVAL` |
+
 | `--models-config-file` | | YAML file containing a list of model backend configs (alias: `--config-file`) | `$LOCALAI_MODELS_CONFIG_FILE`, `$CONFIG_FILE` |
 | `--artifact-download-concurrency` | `1` | How many files of a model artifact to download at once. `1` downloads sequentially. Raising it helps artifacts split into many files on a fast link, at the cost of more concurrent load on the models volume. Whole files only — a single file is never split, so resume and per-file checksum verification are unaffected | `$LOCALAI_ARTIFACT_DOWNLOAD_CONCURRENCY` |
 
+Backend processes receive a private scratch directory through `TMPDIR`, `TMP`,
+and `TEMP`. LocalAI removes that directory when the backend exits and removes
+abandoned directories left by a LocalAI crash before starting another backend.
+Set `$LOCALAI_BACKEND_TEMP_DIR` to choose their base volume. LocalAI always
+appends `localai-UID/backend-runtime`; the default base is `TMPDIR`.
+
 ## Backend Flags
 
 | Parameter | Default | Description | Environment Variable |

gallery/index.yaml

@@ -113,24 +113,7 @@
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
     - https://huggingface.co/unsloth/GLM-5.3-Flash-GGUF
-  description: |
-    # GLM-5.3-Flash
-
-    👋 Join our WeChat or Discord community.
-
-    📖 Check out the GLM-5.3-Flash blog and GLM-5 Technical report.
-
-    📍 Use GLM-5.3-Flash API services on Z.ai API Platform.
-
-    ## Introduction
-
-    We introduce GLM-5.3-Flash, the first natively multimodal model in the GLM-5 series. With 320B total parameters and just 18B active parameters, it outperforms GLM-5.2 across benchmarks and real-world workloads at one-tenth the price, while approaching Claude Opus 4.8 on coding and agentic benchmarks.
-
-    GLM-5.3-Flash starts from a newly trained base model, with its architecture and training recipe redesigned around capability and efficiency. For the first time in the GLM series, we introduce a hybrid architecture combining sparse and linear attention, sharply reducing long-context serving costs while preserving precise long-context capabilities. The model also adopts Manifold-Constrained Hyper-Connections (mHC) to further improve scaling efficiency. Together with our latest 30T-token multimodal pre-training corpus, these changes enable GLM-5.3-Flash to deliver more intelligence with less compute.
-
-    ## Serve GLM-5.3-Flash Locally
-
-    ...
+  description: "# GLM-5.3-Flash\n\n\U0001F44B Join our WeChat or Discord community.\n\n\U0001F4D6 Check out the GLM-5.3-Flash blog and GLM-5 Technical report.\n\n\U0001F4CD Use GLM-5.3-Flash API services on Z.ai API Platform.\n\n## Introduction\n\nWe introduce GLM-5.3-Flash, the first natively multimodal model in the GLM-5 series. With 320B total parameters and just 18B active parameters, it outperforms GLM-5.2 across benchmarks and real-world workloads at one-tenth the price, while approaching Claude Opus 4.8 on coding and agentic benchmarks.\n\nGLM-5.3-Flash starts from a newly trained base model, with its architecture and training recipe redesigned around capability and efficiency. For the first time in the GLM series, we introduce a hybrid architecture combining sparse and linear attention, sharply reducing long-context serving costs while preserving precise long-context capabilities. The model also adopts Manifold-Constrained Hyper-Connections (mHC) to further improve scaling efficiency. Together with our latest 30T-token multimodal pre-training corpus, these changes enable GLM-5.3-Flash to deliver more intelligence with less compute.\n\n## Serve GLM-5.3-Flash Locally\n\n...\n"
   license: "mit"
   tags:
     - llm
@@ -768,9 +751,7 @@
       model: llama-cpp/models/s1-mini/s1-mini-q4_k_m.gguf
       temperature: 0
     system_prompt: >-
-      You are a text normalizer for speech-to-text transcripts. The input begins
-      with a control line specifying the styling, structure, and context settings;
-      clean the transcript to match those settings and output only the cleaned text.
+      You are a text normalizer for speech-to-text transcripts. The input begins with a control line specifying the styling, structure, and context settings; clean the transcript to match those settings and output only the cleaned text.
     template:
       use_tokenizer_template: true
   files:
@@ -796,9 +777,7 @@
       model: llama-cpp/models/s1-mini/s1-mini-f16.gguf
       temperature: 0
     system_prompt: >-
-      You are a text normalizer for speech-to-text transcripts. The input begins
-      with a control line specifying the styling, structure, and context settings;
-      clean the transcript to match those settings and output only the cleaned text.
+      You are a text normalizer for speech-to-text transcripts. The input begins with a control line specifying the styling, structure, and context settings; clean the transcript to match those settings and output only the cleaned text.
     template:
       use_tokenizer_template: true
   files:
@@ -14131,8 +14110,8 @@
       model: kokoro-int8-multi-lang-v1_0/model.int8.onnx
   files:
     - filename: kokoro-int8-multi-lang-v1_0.tar.bz2
-      sha256: 75654a84864be26f345f020f4070c2c019e96dd1b7f9bf6e2ffd59efac6aa5a3
       uri: https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-int8-multi-lang-v1_0.tar.bz2
+      sha256: 4c3052abaa60943a341f193888cf6abd68787dae6ab8ae5c925a706caa247e4e
 - name: supertonic-3
   url: github:mudler/LocalAI/gallery/supertonic.yaml@master
   urls:

pkg/model/initializers.go

@@ -173,7 +173,7 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
 	if !ready {
 		xlog.Debug("GRPC Service NOT ready")
 		startupErr := grpcStartupError(client.Process())
-		stopLoadProcess(client, modelID)
+		ml.stopLoadProcess(client, modelID)
 		return nil, startupErr
 	}
 
@@ -189,11 +189,11 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
 
 	res, err := client.GRPC(o.parallelRequests, ml.wd).LoadModel(o.context, options)
 	if err != nil {
-		stopLoadProcess(client, modelID)
+		ml.stopLoadProcess(client, modelID)
 		return nil, fmt.Errorf("could not load model: %w", err)
 	}
 	if !res.Success {
-		stopLoadProcess(client, modelID)
+		ml.stopLoadProcess(client, modelID)
 		return nil, fmt.Errorf("could not load model (no success): %s", res.Message)
 	}
 
@@ -260,7 +260,7 @@ func lastNonEmptyLine(path string, maxBytes int64) string {
 
 // stopLoadProcess tears down a backend process whose load did not complete.
 // The stop error is only logged: the load error is what the caller reports.
-func stopLoadProcess(client *Model, modelID string) {
+func (ml *ModelLoader) stopLoadProcess(client *Model, modelID string) {
 	process := client.Process()
 	if process == nil {
 		return
@@ -268,6 +268,7 @@ func stopLoadProcess(client *Model, modelID string) {
 	if err := process.Stop(); err != nil {
 		xlog.Warn("failed to stop backend process after failed load", "error", err, "modelID", modelID)
 	}
+	ml.cleanupProcessRuntime(process)
 }
 
 // parallelSlotsFromOptions returns the effective n_parallel from the backend

pkg/model/loader.go

@@ -108,6 +108,10 @@ type ModelLoader struct {
 	// the exit code can't, since a child killed by our own SIGTERM/SIGKILL
 	// reports -1, indistinguishable from a signal-induced crash.
 	stoppingProcs sync.Map
+	// processRuntimes keeps the owned state/scratch directory alive until the
+	// loader has consumed any exit diagnostics. The exit watcher removes the
+	// potentially large scratch contents immediately.
+	processRuntimes sync.Map
 	// loadFailures records, per modelID, the cooldown window applied after a
 	// failed load so that a client repeatedly polling a broken model does not
 	// spawn (and leak) a fresh backend process on every request. Guarded by mu.

pkg/model/process.go

@@ -177,12 +177,14 @@ func (ml *ModelLoader) deleteProcess(ctx context.Context, s string, force bool)
 			// A concurrently crashed/already-reaped process can no longer own
 			// resources even if Stop could not read or signal its PID.
 			store.Delete(s)
+			ml.cleanupProcessRuntime(process)
 			return nil
 		}
 		return err
 	}
 
 	store.Delete(s)
+	ml.cleanupProcessRuntime(process)
 	return nil
 }
 func (ml *ModelLoader) StopGRPC(filter GRPCProcessFilter) error {
@@ -231,16 +233,6 @@ func (ml *ModelLoader) StartProcess(grpcProcess, id string, serverAddress string
 	return ml.startProcess(grpcProcess, id, serverAddress, args...)
 }
 
-// newProcessStateDir creates the directory a backend process uses for its pid,
-// state and log files, and reports why when it cannot.
-func newProcessStateDir() (string, error) {
-	dir, err := os.MkdirTemp(os.TempDir(), "go-processmanager")
-	if err != nil {
-		return "", fmt.Errorf("creating backend process state directory under %s: %w", os.TempDir(), err)
-	}
-	return dir, nil
-}
-
 func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) (*process.Process, error) {
 	// Make sure the process is executable
 	// Check first if it has executable permissions
@@ -262,7 +254,12 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
 		return nil, err
 	}
 
-	env := os.Environ()
+	runtime, err := newBackendProcessRuntime()
+	if err != nil {
+		return nil, err
+	}
+
+	env := backendTempEnvironment(os.Environ(), runtime.tempDir)
 	// Vulkan backends are self-contained: they bundle their own loader and
 	// Mesa driver .so files in lib/ plus the matching ICD manifests in
 	// vulkan/icd.d/. Point the loader at those manifests so it doesn't rely on
@@ -271,16 +268,14 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
 	// and the GPU would silently fall back to CPU). No-op for other backends.
 	env = append(env, vulkanICDEnv(workDir)...)
 
-	// Resolve the state directory here rather than through
+	// Resolve and own the state directory here rather than through
 	// process.WithTemporaryStateDir(). process.New applies its options but
 	// discards the error they return, so a temp directory that cannot be
 	// created leaves StateDir empty and every later option unapplied. Run()
-	// then reported "mkdir : no such file or directory" with no path, hiding
-	// the real cause (a full volume, or a TMPDIR that no longer resolves).
-	stateDir, err := newProcessStateDir()
-	if err != nil {
-		return nil, err
-	}
+	// then reports "mkdir : no such file or directory" with no useful path.
+	// The same owned directory also contains backend scratch so an unexpected
+	// exit cannot strand request files directly in the host's shared /tmp.
+	stateDir := runtime.dir
 
 	grpcControlProcess := process.New(
 		process.WithStateDir(stateDir),
@@ -296,8 +291,10 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
 	}
 
 	if err := grpcControlProcess.Run(); err != nil {
+		runtime.cleanup()
 		return grpcControlProcess, err
 	}
+	ml.processRuntimes.Store(grpcControlProcess, runtime)
 
 	xlog.Debug("GRPC Service state dir", "dir", grpcControlProcess.StateDir())
 
@@ -376,11 +373,35 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string
 			}
 			xlog.Warn("Backend process exited unexpectedly", fields...)
 		}
+		runtime.cleanupScratch()
+		close(runtime.diagnosticsDone)
 	}()
 
 	return grpcControlProcess, nil
 }
 
+func (ml *ModelLoader) cleanupProcessRuntime(process *process.Process) {
+	if process == nil {
+		return
+	}
+	value, ok := ml.processRuntimes.LoadAndDelete(process)
+	if !ok {
+		return
+	}
+	runtime := value.(*backendProcessRuntime)
+	go func() {
+		<-runtime.diagnosticsDone
+		runtime.cleanup()
+	}()
+}
+
+// CleanupProcessRuntime releases state and scratch owned by a process started
+// through StartProcess. Callers that supervise processes outside ModelLoader's
+// model store must invoke it after they have consumed exit diagnostics.
+func (ml *ModelLoader) CleanupProcessRuntime(process *process.Process) {
+	ml.cleanupProcessRuntime(process)
+}
+
 // vulkanICDEnv returns environment overrides that point the Vulkan loader at
 // the ICD manifests a backend bundles in <workDir>/vulkan/icd.d. Vulkan
 // backends ship a self-contained stack — their own loader and Mesa driver .so

pkg/model/process_exit_test.go

@@ -13,8 +13,10 @@ import (
 var _ = Describe("backend process exit diagnostics", func() {
 	It("includes the exit code and final stderr line for an unexpected exit", func() {
 		tmpDir := GinkgoT().TempDir()
+		backendTempRoot := filepath.Join(tmpDir, "backend-runtime")
+		GinkgoT().Setenv(backendTempDirEnv, backendTempRoot)
 		backendPath := filepath.Join(tmpDir, "failing-backend")
-		Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\necho 'first diagnostic' >&2\necho 'fatal metal pipeline error' >&2\nexit 42\n"), 0o700)).To(Succeed())
+		Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\nprintf '%s' \"$TMPDIR\" > \"$0.tmpdir\"\necho 'first diagnostic' >&2\necho 'fatal metal pipeline error' >&2\nexit 42\n"), 0o700)).To(Succeed())
 
 		captured := captureLogs(slog.LevelWarn)
 		DeferCleanup(stopCapturingLogs)
@@ -23,10 +25,16 @@ var _ = Describe("backend process exit diagnostics", func() {
 		process, err := loader.startProcess(backendPath, "test-model", "127.0.0.1:65535")
 		Expect(err).ToNot(HaveOccurred())
 		Eventually(process.Done()).Should(BeClosed())
+		backendTemp, err := os.ReadFile(backendPath + ".tmpdir")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(backendTemp)).To(Equal(filepath.Join(process.StateDir(), "tmp")))
+		Eventually(string(backendTemp)).ShouldNot(BeADirectory())
 		Eventually(captured.String).Should(And(
 			ContainSubstring("Backend process exited unexpectedly"),
 			ContainSubstring("exitCode=42"),
 			ContainSubstring(`stderr="fatal metal pipeline error"`),
 		))
+		loader.cleanupProcessRuntime(process)
+		Eventually(process.StateDir()).ShouldNot(BeADirectory())
 	})
 })

pkg/model/process_runtime.go

@@ -0,0 +1,168 @@
+package model
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/gofrs/flock"
+	"github.com/mudler/xlog"
+)
+
+const (
+	backendTempDirEnv       = "LOCALAI_BACKEND_TEMP_DIR"
+	backendRuntimeDirPrefix = "process-"
+	backendRuntimeMarker    = ".localai-backend-runtime"
+	backendRuntimeMagic     = "localai-backend-runtime-v1\n"
+)
+
+// backendProcessRuntime owns both go-processmanager's state and all temporary
+// files created by one backend process. The held lock distinguishes a live
+// runtime from one abandoned when LocalAI was killed or crashed.
+type backendProcessRuntime struct {
+	dir     string
+	tempDir string
+	lock    *flock.Flock
+	scratch sync.Once
+	once    sync.Once
+	// diagnosticsDone closes after the exit watcher has read the state files.
+	diagnosticsDone chan struct{}
+}
+
+func backendRuntimeRoot() string {
+	base := os.TempDir()
+	if configured := os.Getenv(backendTempDirEnv); configured != "" {
+		base = configured
+	}
+	// Always append a LocalAI- and user-specific namespace. Even if an operator
+	// points the configurable base at /tmp, the sweeper never inspects unrelated
+	// process-* directories in that shared parent.
+	return filepath.Join(base, fmt.Sprintf("localai-%d", os.Getuid()), "backend-runtime")
+}
+
+func newBackendProcessRuntime() (*backendProcessRuntime, error) {
+	root := backendRuntimeRoot()
+	if err := os.MkdirAll(root, 0o700); err != nil {
+		return nil, fmt.Errorf("creating backend runtime root %s: %w", root, err)
+	}
+
+	// Serialize sweeping with creation. Otherwise a second LocalAI instance
+	// could observe the new directory in the tiny window before its owner lock
+	// is acquired and mistake it for an abandoned runtime.
+	sweepLock := flock.New(filepath.Join(root, ".sweep.lock"))
+	if err := sweepLock.Lock(); err != nil {
+		return nil, fmt.Errorf("locking backend runtime root %s: %w", root, err)
+	}
+	defer func() {
+		if err := sweepLock.Unlock(); err != nil {
+			xlog.Warn("Failed to unlock backend runtime root", "root", root, "error", err)
+		}
+	}()
+
+	sweepAbandonedBackendRuntimes(root)
+
+	dir, err := os.MkdirTemp(root, backendRuntimeDirPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("creating backend process runtime under %s: %w", root, err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, backendRuntimeMarker), []byte(backendRuntimeMagic), 0o600); err != nil {
+		_ = os.RemoveAll(dir)
+		return nil, fmt.Errorf("marking backend process runtime %s: %w", dir, err)
+	}
+	runtimeLock := flock.New(filepath.Join(dir, ".owner.lock"))
+	if err := runtimeLock.Lock(); err != nil {
+		_ = os.RemoveAll(dir)
+		return nil, fmt.Errorf("locking backend process runtime %s: %w", dir, err)
+	}
+	tempDir := filepath.Join(dir, "tmp")
+	if err := os.Mkdir(tempDir, 0o700); err != nil {
+		_ = runtimeLock.Unlock()
+		_ = os.RemoveAll(dir)
+		return nil, fmt.Errorf("creating backend scratch directory %s: %w", tempDir, err)
+	}
+
+	return &backendProcessRuntime{
+		dir:             dir,
+		tempDir:         tempDir,
+		lock:            runtimeLock,
+		diagnosticsDone: make(chan struct{}),
+	}, nil
+}
+
+func sweepAbandonedBackendRuntimes(root string) {
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		xlog.Warn("Failed to inspect backend runtime root", "root", root, "error", err)
+		return
+	}
+	for _, entry := range entries {
+		if !entry.IsDir() || !strings.HasPrefix(entry.Name(), backendRuntimeDirPrefix) {
+			continue
+		}
+		dir := filepath.Join(root, entry.Name())
+		marker, err := os.ReadFile(filepath.Join(dir, backendRuntimeMarker))
+		if err != nil || string(marker) != backendRuntimeMagic {
+			continue
+		}
+		ownerLock := flock.New(filepath.Join(dir, ".owner.lock"))
+		available, err := ownerLock.TryLock()
+		if err != nil {
+			xlog.Warn("Failed to inspect backend runtime ownership", "dir", dir, "error", err)
+			continue
+		}
+		if !available {
+			continue
+		}
+		if err := ownerLock.Unlock(); err != nil {
+			xlog.Warn("Failed to release abandoned backend runtime lock", "dir", dir, "error", err)
+			continue
+		}
+		if err := os.RemoveAll(dir); err != nil {
+			xlog.Warn("Failed to remove abandoned backend runtime", "dir", dir, "error", err)
+		}
+	}
+}
+
+func (r *backendProcessRuntime) cleanup() {
+	if r == nil {
+		return
+	}
+	r.once.Do(func() {
+		r.cleanupScratch()
+		if err := r.lock.Unlock(); err != nil {
+			xlog.Warn("Failed to unlock backend process runtime", "dir", r.dir, "error", err)
+		}
+		if err := os.RemoveAll(r.dir); err != nil {
+			xlog.Warn("Failed to remove backend process runtime", "dir", r.dir, "error", err)
+		}
+	})
+}
+
+func (r *backendProcessRuntime) cleanupScratch() {
+	if r == nil {
+		return
+	}
+	r.scratch.Do(func() {
+		if err := os.RemoveAll(r.tempDir); err != nil {
+			xlog.Warn("Failed to remove backend scratch directory", "dir", r.tempDir, "error", err)
+		}
+	})
+}
+
+func backendTempEnvironment(env []string, tempDir string) []string {
+	result := make([]string, 0, len(env)+3)
+	for _, entry := range env {
+		key, _, found := strings.Cut(entry, "=")
+		if found && (key == "TMPDIR" || key == "TMP" || key == "TEMP") {
+			continue
+		}
+		result = append(result, entry)
+	}
+	return append(result,
+		"TMPDIR="+tempDir,
+		"TMP="+tempDir,
+		"TEMP="+tempDir,
+	)
+}

pkg/model/process_runtime_test.go

@@ -0,0 +1,109 @@
+package model
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Backend process runtime directory", func() {
+	It("keeps active runtimes while sweeping abandoned ones", func() {
+		root := GinkgoT().TempDir()
+		GinkgoT().Setenv(backendTempDirEnv, root)
+		ownedRoot := backendRuntimeRoot()
+		unrelated := filepath.Join(root, backendRuntimeDirPrefix+"unrelated")
+		Expect(os.MkdirAll(unrelated, 0o700)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(unrelated, "keep"), []byte("unrelated"), 0o600)).To(Succeed())
+
+		active, err := newBackendProcessRuntime()
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(active.cleanup)
+		Expect(os.WriteFile(filepath.Join(active.tempDir, "active.img"), []byte("active"), 0o600)).To(Succeed())
+
+		abandoned := filepath.Join(ownedRoot, backendRuntimeDirPrefix+"abandoned")
+		Expect(os.MkdirAll(abandoned, 0o700)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(abandoned, backendRuntimeMarker), []byte(backendRuntimeMagic), 0o600)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(abandoned, "orphan.img"), []byte("orphan"), 0o600)).To(Succeed())
+		foreign := filepath.Join(ownedRoot, backendRuntimeDirPrefix+"foreign")
+		Expect(os.MkdirAll(foreign, 0o700)).To(Succeed())
+		Expect(os.WriteFile(filepath.Join(foreign, "keep"), []byte("foreign"), 0o600)).To(Succeed())
+
+		other, err := newBackendProcessRuntime()
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(other.cleanup)
+
+		Expect(active.dir).To(BeADirectory())
+		Expect(abandoned).ToNot(BeAnExistingFile())
+		Expect(foreign).To(BeADirectory())
+		Expect(unrelated).To(BeADirectory())
+	})
+
+	It("uses one private directory for process state and backend scratch", func() {
+		root := GinkgoT().TempDir()
+		GinkgoT().Setenv(backendTempDirEnv, root)
+
+		runtime, err := newBackendProcessRuntime()
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(runtime.cleanup)
+
+		Expect(filepath.Dir(runtime.dir)).To(Equal(backendRuntimeRoot()))
+		Expect(runtime.tempDir).To(Equal(filepath.Join(runtime.dir, "tmp")))
+		info, err := os.Stat(runtime.tempDir)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(info.IsDir()).To(BeTrue())
+		Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o700)))
+	})
+
+	It("overrides inherited temp variables for the backend only", func() {
+		env := backendTempEnvironment([]string{
+			"PATH=/bin",
+			"TMPDIR=/old/tmpdir",
+			"TMP=/old/tmp",
+			"TEMP=/old/temp",
+		}, "/owned/scratch")
+
+		Expect(env).To(ConsistOf(
+			"PATH=/bin",
+			"TMPDIR=/owned/scratch",
+			"TMP=/owned/scratch",
+			"TEMP=/owned/scratch",
+		))
+		for _, key := range []string{"TMPDIR", "TMP", "TEMP"} {
+			count := 0
+			for _, entry := range env {
+				if strings.HasPrefix(entry, key+"=") {
+					count++
+				}
+			}
+			Expect(count).To(Equal(1), key)
+		}
+	})
+
+	It("removes the runtime when its owner exits", func() {
+		GinkgoT().Setenv(backendTempDirEnv, GinkgoT().TempDir())
+
+		runtime, err := newBackendProcessRuntime()
+		Expect(err).ToNot(HaveOccurred())
+		dir := runtime.dir
+		runtime.cleanup()
+
+		Expect(dir).ToNot(BeAnExistingFile())
+	})
+
+	It("reports which configured root cannot be used", func() {
+		parent := GinkgoT().TempDir()
+		file := filepath.Join(parent, "not-a-directory")
+		Expect(os.WriteFile(file, []byte("x"), 0o600)).To(Succeed())
+		base := filepath.Join(file, "backend-runtime")
+		GinkgoT().Setenv(backendTempDirEnv, base)
+		root := backendRuntimeRoot()
+
+		runtime, err := newBackendProcessRuntime()
+		Expect(err).To(HaveOccurred())
+		Expect(runtime).To(BeNil())
+		Expect(err.Error()).To(ContainSubstring(root))
+	})
+})

pkg/model/process_statedir_test.go

@@ -1,38 +0,0 @@
-package model
-
-import (
-	"os"
-	"path/filepath"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Backend process state directory", func() {
-	It("reports why the state directory could not be created", func() {
-		// A worker whose volume is full, or whose TMPDIR no longer resolves,
-		// cannot get a state directory. go-processmanager's New() drops the
-		// option error, leaving StateDir empty, and Run() then failed with
-		// "mkdir : no such file or directory" naming no path at all. Resolving
-		// the directory here keeps the real cause attached.
-		GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "does-not-exist"))
-
-		dir, err := newProcessStateDir()
-		Expect(err).To(HaveOccurred())
-		Expect(dir).To(BeEmpty())
-		Expect(err.Error()).To(ContainSubstring("backend process state directory"))
-		Expect(err.Error()).To(ContainSubstring("does-not-exist"),
-			"the error must name the directory it could not create")
-	})
-
-	It("returns a usable directory when the temp location works", func() {
-		GinkgoT().Setenv("TMPDIR", GinkgoT().TempDir())
-
-		dir, err := newProcessStateDir()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(dir).ToNot(BeEmpty())
-		info, statErr := os.Stat(dir)
-		Expect(statErr).ToNot(HaveOccurred())
-		Expect(info.IsDir()).To(BeTrue())
-	})
-})