fix(backends): bound temporary scratch files (#11941)

Backend processes shared the host temporary directory, so crashes could leave request images and audio behind until the filesystem filled. Give each process a locked LocalAI-owned runtime, remove scratch on exit, and sweep only marked abandoned runtimes at the next start.

Also close known request error-path leaks in the Python media backends, CrispASR, LongCat Video, and stable-diffusion.cpp.

Assisted-by: Codex:gpt-5

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/python/chatterbox/backend.py

@@ -19,6 +19,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
+from temp_utils import cleanup_paths
 
 import tempfile
 
@@ -115,11 +116,6 @@ def merge_audio_files(audio_files, output_path, sample_rate):
     # Save the merged audio
     ta.save(output_path, merged_waveform, sample_rate)
     
-    # Clean up temporary files
-    for audio_file in audio_files:
-        if os.path.exists(audio_file):
-            os.remove(audio_file)
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
@@ -226,19 +222,20 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 text_chunks = split_text_at_word_boundary(request.text, max_length=250)
                 print(f"Splitting text into chunks of 250 characters: {len(text_chunks)}", file=sys.stderr)
                 # Generate audio for each chunk
-                temp_audio_files = []
-                for i, chunk in enumerate(text_chunks):
-                    # Generate audio for this chunk
-                    wav = self.model.generate(chunk, **kwargs)
-                    
-                    # Create temporary file for this chunk
-                    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
-                    temp_file.close()
-                    ta.save(temp_file.name, wav, self.model.sr)
-                    temp_audio_files.append(temp_file.name)
-                
-                # Merge all audio files
-                merge_audio_files(temp_audio_files, request.dst, self.model.sr)
+                with cleanup_paths() as temp_audio_files:
+                    for i, chunk in enumerate(text_chunks):
+                        # Generate audio for this chunk
+                        wav = self.model.generate(chunk, **kwargs)
+
+                        # Register ownership before saving so a partial write is
+                        # removed too when generation or encoding fails.
+                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
+                        temp_file.close()
+                        temp_audio_files.append(temp_file.name)
+                        ta.save(temp_file.name, wav, self.model.sr)
+
+                    # Merge all audio files
+                    merge_audio_files(temp_audio_files, request.dst, self.model.sr)
             else:
                 # Generate audio using ChatterboxTTS for short text
                 wav = self.model.generate(request.text, **kwargs)

backend/python/common/temp_utils.py

@@ -0,0 +1,36 @@
+import base64
+import contextlib
+import os
+import tempfile
+
+
+@contextlib.contextmanager
+def materialize_base64(data, suffix=""):
+    """Materialize base64 data for a path-only library and always remove it."""
+    descriptor, path = tempfile.mkstemp(prefix="localai-media-", suffix=suffix)
+    try:
+        with os.fdopen(descriptor, "wb") as output:
+            descriptor = None
+            output.write(base64.b64decode(data))
+        yield path
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+
+
+@contextlib.contextmanager
+def cleanup_paths():
+    """Collect temporary paths and remove them on success or failure."""
+    paths = []
+    try:
+        yield paths
+    finally:
+        for path in paths:
+            try:
+                os.remove(path)
+            except OSError:
+                pass

backend/python/common/temp_utils_test.py

@@ -0,0 +1,41 @@
+import os
+import tempfile
+import unittest
+from unittest import mock
+
+from temp_utils import cleanup_paths, materialize_base64
+
+
+class MaterializeBase64Test(unittest.TestCase):
+    def test_removes_materialized_file_after_success(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with mock.patch.object(tempfile, "tempdir", directory):
+                with materialize_base64("aGVsbG8=", suffix=".data") as path:
+                    with open(path, "rb") as materialized:
+                        self.assertEqual(materialized.read(), b"hello")
+                self.assertFalse(os.path.exists(path))
+
+    def test_removes_materialized_file_when_consumer_fails(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with mock.patch.object(tempfile, "tempdir", directory):
+                with self.assertRaisesRegex(RuntimeError, "decode failed"):
+                    with materialize_base64("aGVsbG8="):
+                        raise RuntimeError("decode failed")
+                self.assertEqual(os.listdir(directory), [])
+
+
+class CleanupPathsTest(unittest.TestCase):
+    def test_removes_every_registered_path_after_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            paths = [os.path.join(directory, name) for name in ("one.wav", "two.wav")]
+            with self.assertRaisesRegex(RuntimeError, "merge failed"):
+                with cleanup_paths() as registered:
+                    for path in paths:
+                        open(path, "wb").close()
+                        registered.append(path)
+                    raise RuntimeError("merge failed")
+            self.assertEqual(os.listdir(directory), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/longcat-video/backend.py

@@ -6,6 +6,7 @@ import datetime
 import gc
 import math
 import os
+import shutil
 import signal
 import subprocess
 import sys
@@ -888,6 +889,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
     def _release_model(self):
         self.pipeline = None
         self.model_kind = None
+        try:
+            if hasattr(self, "dist") and self.dist.is_initialized():
+                self.dist.destroy_process_group()
+        finally:
+            if self._dist_store_dir is not None:
+                shutil.rmtree(self._dist_store_dir, ignore_errors=True)
+                self._dist_store_dir = None
         gc.collect()
         if hasattr(self, "torch") and self.torch.cuda.is_available():
             self.torch.cuda.empty_cache()

backend/python/vllm-omni/backend.py

@@ -19,7 +19,6 @@ import base64
 import io
 import json
 import gc
-import tempfile
 
 from PIL import Image
 import torch
@@ -34,6 +33,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from temp_utils import materialize_base64
 from vllm_utils import parse_options, messages_to_dicts, setup_parsers
 
 
@@ -118,13 +118,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return video_to_ndarrays(video_path, num_frames=16)
         # Try base64 decode
         try:
-            timestamp = str(int(time.time() * 1000))
-            p = os.path.join(tempfile.gettempdir(), f"vl-{timestamp}.data")
-            with open(p, "wb") as f:
-                f.write(base64.b64decode(video_path))
-            video = VideoAsset(name=p).np_ndarrays
-            os.remove(p)
-            return video
+            with materialize_base64(video_path, suffix=".data") as path:
+                return VideoAsset(name=path).np_ndarrays
         except:
             return None
 
@@ -136,15 +131,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return (audio_signal.astype(np.float32), sr)
         # Try base64 decode
         try:
-            audio_data = base64.b64decode(audio_path)
-            # Save to temp file and load
-            timestamp = str(int(time.time() * 1000))
-            p = os.path.join(tempfile.gettempdir(), f"audio-{timestamp}.wav")
-            with open(p, "wb") as f:
-                f.write(audio_data)
-            audio_signal, sr = librosa.load(p, sr=16000)
-            os.remove(p)
-            return (audio_signal.astype(np.float32), sr)
+            with materialize_base64(audio_path, suffix=".wav") as path:
+                audio_signal, sr = librosa.load(path, sr=16000)
+                return (audio_signal.astype(np.float32), sr)
         except:
             return None
 

backend/python/vllm/backend.py

@@ -10,7 +10,6 @@ import os
 import json
 import time
 import gc
-import tempfile
 from typing import List
 from PIL import Image
 
@@ -23,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from python_utils import attach_media_parts
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from temp_utils import materialize_base64
 from vllm_utils import apply_options_to_engine_args, normalize_option_key
 
 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -1005,13 +1005,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             Video: The loaded video.
         """
         try:
-            timestamp = str(int(time.time() * 1000))  # Generate timestamp
-            p = os.path.join(tempfile.gettempdir(), f"vl-{timestamp}.data")
-            with open(p, "wb") as f:
-                f.write(base64.b64decode(video_path))
-            video = VideoAsset(name=p).np_ndarrays
-            os.remove(p)
-            return video
+            with materialize_base64(video_path, suffix=".data") as path:
+                return VideoAsset(name=path).np_ndarrays
         except Exception as e:
             print(f"Error loading video {video_path}: {e}", file=sys.stderr)
             return None