feat(backends): add LongCat video and avatar generation (#10792)

* feat(backends): add LongCat video and avatar generation Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] [web] * refactor(config): declare model I/O modalities Make model configs declare input and output modalities so capability discovery no longer branches on backend or checkpoint names. Complete the LongCat gallery and user documentation, make the SDPA patch apply to the pinned upstream revision, and stabilize the Agent Jobs race exposed by the required hook. Assisted-by: Codex:GPT-5 [web] --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-07-30 09:57:57 -04:00 · 2026-07-12 23:58:46 +02:00
parent af8f74cba2
commit b00422e45f
72 changed files with 2784 additions and 190 deletions
--- a/.github/backend-matrix.yml
+++ b/.github/backend-matrix.yml
@@ -478,6 +478,19 @@ include:
    dockerfile: "./backend/Dockerfile.python"
    context: "./"
    ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
  - build-type: 'cublas'
    cuda-major-version: "12"
    cuda-minor-version: "8"
@@ -1149,6 +1162,19 @@ include:
    dockerfile: "./backend/Dockerfile.python"
    context: "./"
    ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
  - build-type: 'cublas'
    cuda-major-version: "13"
    cuda-minor-version: "0"
@@ -1357,6 +1383,19 @@ include:
    backend: "vllm-omni"
    dockerfile: "./backend/Dockerfile.python"
    context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
  - build-type: 'l4t'
    cuda-major-version: "13"
    cuda-minor-version: "0"
--- a/.github/workflows/bump_deps.yaml
+++ b/.github/workflows/bump_deps.yaml
@@ -26,6 +26,10 @@ jobs:
            variable: "DS4_VERSION"
            branch: "main"
            file: "backend/cpp/ds4/Makefile"
+          - repository: "meituan-longcat/LongCat-Video"
+            variable: "LONGCAT_VIDEO_VERSION"
+            branch: "main"
+            file: "backend/python/longcat-video/Makefile"
          - repository: "localai-org/privacy-filter.cpp"
            variable: "PRIVACY_FILTER_VERSION"
            branch: "master"
--- a/8
+++ b/8
@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin

 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -565,6 +565,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
+	$(MAKE) -C backend/python/longcat-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -594,6 +595,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
+	$(MAKE) -C backend/python/longcat-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -1254,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true
 BACKEND_KOKORO = kokoro|python|.|false|true
 BACKEND_VLLM = vllm|python|.|false|true
 BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
+BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true
 BACKEND_SGLANG = sglang|python|.|false|true
 BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
 BACKEND_CHATTERBOX = chatterbox|python|.|false|true
@@ -1339,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
+$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
@@ -1375,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar

-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter

 ########################################################
 ### Mock Backend for E2E Tests
--- a/backend/README.md
+++ b/backend/README.md
@@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **vllm**: High-performance LLM inference
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
+- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
 - **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro
--- a/backend/backend.proto
+++ b/backend/backend.proto
@@ -577,6 +577,10 @@ message GenerateVideoRequest {
  float cfg_scale = 10;    // Classifier-free guidance scale
  int32 step = 11;         // Number of inference steps
  string dst = 12;        // Output path for the generated video
+  string audio = 13;      // Path to staged audio for audio-conditioned video
+  // Backend-specific per-request generation parameters. Values are strings
+  // and are validated/coerced by the selected backend.
+  map<string, string> params = 14;
 }

 message TTSRequest {
@@ -1256,4 +1260,3 @@ message ForwardReply {
  repeated ForwardHeader headers = 2;
  bytes body_chunk = 3;
 }
-
--- a/backend/index.yaml
+++ b/backend/index.yaml
@@ -824,6 +824,30 @@
    nvidia-cuda-12: "cuda12-vllm-omni"
    nvidia-cuda-13: "cuda13-vllm-omni"
    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-omni"
+- &longcat-video
+  name: "longcat-video"
+  alias: "longcat-video"
+  license: mit
+  urls:
+    - https://github.com/meituan-longcat/LongCat-Video
+  tags:
+    - text-to-video
+    - image-to-video
+    - audio-to-video
+    - avatar-generation
+    - video-generation
+    - CUDA
+  icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg
+  description: |
+    LongCat-Video generation for text, image, and audio-conditioned avatars.
+    Supports LongCat-Video and LongCat-Video-Avatar-1.5, including multi-segment
+    talking-head continuation and an SDPA path for NVIDIA Blackwell ARM64 systems.
+    Requires Linux with an NVIDIA CUDA GPU; CPU, ROCm, and macOS are unsupported.
+  capabilities:
+    nvidia: "cuda12-longcat-video"
+    nvidia-cuda-12: "cuda12-longcat-video"
+    nvidia-cuda-13: "cuda13-longcat-video"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video"
 - &mlx
  name: "mlx"
  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -3605,6 +3629,44 @@
  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm-omni"
  mirrors:
    - localai/localai-backends:master-gpu-rocm-hipblas-vllm-omni
+# longcat-video
+- !!merge <<: *longcat-video
+  name: "longcat-video-development"
+  capabilities:
+    nvidia: "cuda12-longcat-video-development"
+    nvidia-cuda-12: "cuda12-longcat-video-development"
+    nvidia-cuda-13: "cuda13-longcat-video-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video
 # rfdetr
 - !!merge <<: *rfdetr
  name: "rfdetr-development"
--- a/backend/python/README.md
+++ b/backend/python/README.md
@@ -27,6 +27,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro

 ### Computer Vision
 - **diffusers** - Stable Diffusion and image generation
+- **longcat-video** - CUDA video and speech-driven avatar generation with LongCat-Video
 - **mlx-vlm** - Vision-language models for Apple Silicon
 - **rfdetr** - Object detection models

--- a/backend/python/longcat-video/.gitignore
+++ b/backend/python/longcat-video/.gitignore
@@ -0,0 +1,6 @@
+backend_pb2.py
+backend_pb2_grpc.py
+lib/
+python/
+sources/
+venv/
--- a/backend/python/longcat-video/Makefile
+++ b/backend/python/longcat-video/Makefile
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+
+LONGCAT_VIDEO_VERSION?=6b3f4b8582a8bc3f20f795735f5383716c4ba794
+LONGCAT_VIDEO_REPO?=https://github.com/meituan-longcat/LongCat-Video
+LONGCAT_SOURCE_STAMP=sources/LongCat-Video/.localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: all
+all: $(LONGCAT_SOURCE_STAMP)
+	bash install.sh
+
+$(LONGCAT_SOURCE_STAMP): patches/0001-sdpa-attention-fallback.patch
+	rm -rf sources/LongCat-Video
+	mkdir -p sources/LongCat-Video
+	cd sources/LongCat-Video && git init -q && \
+		git remote add origin $(LONGCAT_VIDEO_REPO) && \
+		git fetch --depth 1 origin $(LONGCAT_VIDEO_VERSION) && \
+		git checkout --detach FETCH_HEAD && \
+		git apply ../../patches/0001-sdpa-attention-fallback.patch && \
+		rm -rf .git && \
+		touch .localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: run
+run: all
+	bash run.sh
+
+.PHONY: test
+test: all
+	bash test.sh
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2.py backend_pb2_grpc.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf __pycache__ lib python sources venv
--- a/backend/python/longcat-video/README.md
+++ b/backend/python/longcat-video/README.md
@@ -0,0 +1,44 @@
+# LongCat Video backend
+
+This backend serves Meituan's `LongCat-Video` and
+`LongCat-Video-Avatar-1.5` checkpoints through LocalAI's `GenerateVideo`
+RPC. It supports:
+
+- text-to-video and image-to-video with `LongCat-Video`;
+- audio + text-to-avatar and portrait + audio-to-avatar with Avatar 1.5;
+- multi-segment avatar continuation for speech longer than one segment;
+- PyTorch SDPA when FlashAttention is unavailable, including CUDA 13 ARM64
+  systems such as NVIDIA DGX Spark.
+
+Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the
+LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md)
+for Studio and API examples, hardware requirements, and manual configuration.
+
+The upstream source is pinned in `Makefile` and patched at build time. The
+patch adds only the missing SDPA attention branches; model and source licenses
+remain MIT.
+
+## Model options
+
+| Option | Default | Description |
+| --- | --- | --- |
+| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`. The packaged backend guarantees only `sdpa`. |
+| `use_distill` | `true` for Avatar, `false` for base | Loads the checkpoint's fast distillation LoRA. |
+| `use_int8` | `false` | Loads Avatar 1.5's INT8 DiT. BF16 has a lower load-time peak on unified-memory systems. |
+| `base_model` | `meituan-longcat/LongCat-Video` | Base components used by Avatar 1.5. |
+| `max_segments` | `8` | Maximum avatar continuation segments accepted per request. |
+| `resolution` | `480p` | Image-conditioned generation resolution (`480p` or `720p`). |
+
+Per-request `params` may set `num_segments`, `audio_guidance_scale`,
+`offload_kv_cache`, `ref_img_index`, `mask_frame_range`, and `resolution`.
+
+Gallery and imported configs declare `known_input_modalities` and
+`known_output_modalities`. Keep those declarations in manual configs as well;
+they let model discovery distinguish base image-conditioned video from Avatar
+audio conditioning without inspecting the backend or checkpoint name.
+
+LongCat is CUDA-only and very large. Avatar 1.5 also loads tokenizer,
+text-encoder, and VAE components from the base checkpoint. Keep ample unified
+memory and storage available; no CPU or macOS backend image is published. The
+initial backend supports one GPU per process; tensor parallel sizes above one
+are rejected explicitly.
--- a/backend/python/longcat-video/backend.py
+++ b/backend/python/longcat-video/backend.py
@@ -0,0 +1,904 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+import datetime
+import gc
+import math
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import traceback
+from concurrent import futures
+
+import grpc
+
+import backend_pb2
+import backend_pb2_grpc
+
+from longcat_utils import (
+    BASE_MODEL_ID,
+    MODEL_KIND_AVATAR,
+    MODEL_KIND_BASE,
+    attention_overrides,
+    avatar_segments_for_duration,
+    avatar_segments_for_frames,
+    classify_model,
+    normalize_model_source,
+    normalize_num_frames,
+    parse_options,
+    require_bool,
+    require_float,
+    require_int,
+    validate_dimensions,
+)
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "sources", "LongCat-Video"))
+
+from grpc_auth import get_auth_interceptors
+
+
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+
+DEFAULT_NEGATIVE_PROMPT = (
+    "Close-up, bright tones, overexposed, static, blurred details, subtitles, "
+    "paintings, low quality, JPEG compression residue, ugly, incomplete, extra "
+    "fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, "
+    "misshapen limbs, fused fingers, still picture, messy background, three legs, "
+    "many people in the background, walking backwards"
+)
+
+LOAD_OPTIONS = {
+    "attention_backend",
+    "base_model",
+    "max_segments",
+    "resolution",
+    "use_distill",
+    "use_int8",
+}
+
+REQUEST_PARAMS = {
+    "audio_guidance_scale",
+    "mask_frame_range",
+    "num_segments",
+    "offload_kv_cache",
+    "ref_img_index",
+    "resolution",
+}
+
+BASE_CHECKPOINT_PATTERNS = [
+    "config.json",
+    "model_index.json",
+    "dit/**",
+    "lora/cfg_step_lora.safetensors",
+    "scheduler/**",
+    "text_encoder/**",
+    "tokenizer/**",
+    "vae/**",
+]
+
+AVATAR_BASE_PATTERNS = [
+    "config.json",
+    "model_index.json",
+    "text_encoder/**",
+    "tokenizer/**",
+    "vae/**",
+]
+
+AVATAR_COMMON_PATTERNS = [
+    "config.json",
+    "lora/dmd_lora.safetensors",
+    "model_index.json",
+    "scheduler/**",
+    "whisper-large-v3/config.json",
+    "whisper-large-v3/model.safetensors",
+    "whisper-large-v3/preprocessor_config.json",
+]
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.model_kind = None
+        self.pipeline = None
+        self.options = {}
+        self.device_index = 0
+        self.cp_split_hw = None
+        self._dist_store_dir = None
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        model = request.Model
+        if request.ModelFile and os.path.isdir(request.ModelFile):
+            model = request.ModelFile
+
+        model_kind = classify_model(model)
+        if model_kind is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "longcat-video only accepts LongCat-Video or LongCat-Video-Avatar-1.5 checkpoints",
+            )
+
+        try:
+            options = parse_options(request.Options)
+            unknown = sorted(set(options) - LOAD_OPTIONS)
+            if unknown:
+                raise ValueError(f"unknown model option(s): {', '.join(unknown)}")
+
+            self._import_torch()
+            if not self.torch.cuda.is_available():
+                return self._fail(
+                    context,
+                    grpc.StatusCode.FAILED_PRECONDITION,
+                    "longcat-video requires an NVIDIA CUDA GPU",
+                )
+            if request.TensorParallelSize > 1:
+                return self._fail(
+                    context,
+                    grpc.StatusCode.UNIMPLEMENTED,
+                    "longcat-video currently supports one GPU per backend process",
+                )
+            self._import_runtime()
+
+            attention_name = str(options.get("attention_backend", "sdpa")).lower()
+            attention_overrides(attention_name)
+            resolution = str(options.get("resolution", "480p")).lower()
+            if resolution not in {"480p", "720p"}:
+                raise ValueError("resolution must be 480p or 720p")
+
+            use_distill_default = model_kind == MODEL_KIND_AVATAR
+            use_distill = require_bool(
+                options.get("use_distill", use_distill_default),
+                "use_distill",
+            )
+            use_int8 = require_bool(options.get("use_int8", False), "use_int8")
+            if model_kind == MODEL_KIND_BASE and use_int8:
+                raise ValueError(
+                    "use_int8 is supported only by LongCat-Video-Avatar-1.5"
+                )
+
+            self.options = {
+                **options,
+                "attention_backend": attention_name,
+                "resolution": resolution,
+                "use_distill": use_distill,
+                "use_int8": use_int8,
+                "max_segments": require_int(
+                    options.get("max_segments", 8),
+                    "max_segments",
+                    minimum=1,
+                    maximum=64,
+                ),
+            }
+
+            self._release_model()
+            self._ensure_distributed()
+            if model_kind == MODEL_KIND_BASE:
+                self._load_base_model(model)
+            else:
+                self._load_avatar_model(model)
+            self.model_kind = model_kind
+            print(
+                f"Loaded {normalize_model_source(model)} as {model_kind} "
+                f"with attention_backend={attention_name}",
+                file=sys.stderr,
+            )
+            return backend_pb2.Result(message="Model loaded successfully", success=True)
+        except ValueError as err:
+            self._release_model()
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            self._release_model()
+            print(f"Error loading LongCat model: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"failed to load LongCat model: {err}",
+            )
+
+    def Free(self, request, context):
+        self._release_model()
+        return backend_pb2.Result(message="Model released", success=True)
+
+    def GenerateVideo(self, request, context):
+        if self.pipeline is None or self.model_kind is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.FAILED_PRECONDITION,
+                "model is not loaded",
+            )
+        if not request.prompt.strip():
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "prompt is required",
+            )
+        if not request.dst:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "output destination is required",
+            )
+        if request.end_image:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "longcat-video does not support end_image conditioning",
+            )
+
+        request_state = {"finished": False}
+
+        def interrupt_if_cancelled():
+            if not request_state["finished"] and self.pipeline is not None:
+                self.pipeline._interrupt = True
+
+        try:
+            params = dict(request.params)
+            unknown = sorted(set(params) - REQUEST_PARAMS)
+            if unknown:
+                raise ValueError(f"unknown request param(s): {', '.join(unknown)}")
+
+            os.makedirs(os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True)
+            if hasattr(context, "add_callback"):
+                context.add_callback(interrupt_if_cancelled)
+
+            if request.start_image and not os.path.isfile(request.start_image):
+                raise ValueError("start_image is not a readable staged file")
+            if request.num_frames < 0:
+                raise ValueError("num_frames must not be negative")
+
+            if self.model_kind == MODEL_KIND_BASE:
+                if request.audio:
+                    raise ValueError(
+                        "audio input requires a LongCat-Video-Avatar-1.5 model"
+                    )
+                self._generate_base(request, params)
+            else:
+                self._generate_avatar(request, params, context)
+
+            return backend_pb2.Result(
+                message="Video generated successfully", success=True
+            )
+        except ValueError as err:
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            print(f"Error generating LongCat video: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"LongCat video generation failed: {err}",
+            )
+        finally:
+            request_state["finished"] = True
+            if self.pipeline is not None:
+                self.pipeline._interrupt = False
+
+    def _import_torch(self):
+        if hasattr(self, "torch"):
+            return
+
+        import torch
+
+        self.torch = torch
+
+    def _import_runtime(self):
+        if hasattr(self, "LongCatVideoPipeline"):
+            return
+
+        import imageio.v2 as imageio
+        import imageio_ffmpeg
+        import librosa
+        import numpy as np
+        import torch.distributed as dist
+        from diffusers.utils import load_image
+        from huggingface_hub import snapshot_download
+        from PIL import Image
+        from transformers import AutoTokenizer, UMT5EncoderModel
+
+        from longcat_video.audio_process import (
+            get_audio_encoder,
+            get_audio_feature_extractor,
+        )
+        from longcat_video.context_parallel import context_parallel_util
+        from longcat_video.modules.autoencoder_kl_wan import AutoencoderKLWan
+        from longcat_video.modules.avatar.longcat_video_dit_avatar import (
+            LongCatVideoAvatarTransformer3DModel,
+        )
+        from longcat_video.modules.longcat_video_dit import (
+            LongCatVideoTransformer3DModel,
+        )
+        from longcat_video.modules.quantization import load_quantized_dit
+        from longcat_video.modules.scheduling_flow_match_euler_discrete import (
+            FlowMatchEulerDiscreteScheduler,
+        )
+        from longcat_video.pipeline_longcat_video import LongCatVideoPipeline
+        from longcat_video.pipeline_longcat_video_avatar import (
+            LongCatVideoAvatarPipeline,
+        )
+
+        self.imageio = imageio
+        self.imageio_ffmpeg = imageio_ffmpeg
+        self.librosa = librosa
+        self.np = np
+        self.dist = dist
+        self.load_image = load_image
+        self.snapshot_download = snapshot_download
+        self.Image = Image
+        self.AutoTokenizer = AutoTokenizer
+        self.UMT5EncoderModel = UMT5EncoderModel
+        self.get_audio_encoder = get_audio_encoder
+        self.get_audio_feature_extractor = get_audio_feature_extractor
+        self.context_parallel_util = context_parallel_util
+        self.AutoencoderKLWan = AutoencoderKLWan
+        self.LongCatVideoAvatarTransformer3DModel = LongCatVideoAvatarTransformer3DModel
+        self.LongCatVideoTransformer3DModel = LongCatVideoTransformer3DModel
+        self.load_quantized_dit = load_quantized_dit
+        self.FlowMatchEulerDiscreteScheduler = FlowMatchEulerDiscreteScheduler
+        self.LongCatVideoPipeline = LongCatVideoPipeline
+        self.LongCatVideoAvatarPipeline = LongCatVideoAvatarPipeline
+
+    def _ensure_distributed(self):
+        self.torch.cuda.set_device(self.device_index)
+        if not self.dist.is_initialized():
+            self._dist_store_dir = tempfile.mkdtemp(prefix="localai-longcat-dist-")
+            init_file = os.path.join(self._dist_store_dir, "store")
+            self.dist.init_process_group(
+                backend="nccl",
+                init_method=f"file://{init_file}",
+                rank=0,
+                world_size=1,
+                timeout=datetime.timedelta(hours=24),
+            )
+            self.context_parallel_util.init_context_parallel(
+                context_parallel_size=1,
+                global_rank=0,
+                world_size=1,
+            )
+            self.cp_split_hw = self.context_parallel_util.get_optimal_split(1)
+
+    def _resolve_checkpoint(self, model, patterns):
+        source = normalize_model_source(model)
+        if os.path.isdir(source):
+            return source
+        print(f"Downloading required files for {source}", file=sys.stderr)
+        return self.snapshot_download(repo_id=source, allow_patterns=patterns)
+
+    def _load_base_model(self, model):
+        checkpoint = self._resolve_checkpoint(model, BASE_CHECKPOINT_PATTERNS)
+        dtype = self.torch.bfloat16
+        overrides = attention_overrides(self.options["attention_backend"])
+
+        tokenizer = self.AutoTokenizer.from_pretrained(
+            checkpoint,
+            subfolder="tokenizer",
+        )
+        text_encoder = self.UMT5EncoderModel.from_pretrained(
+            checkpoint,
+            subfolder="text_encoder",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        vae = self.AutoencoderKLWan.from_pretrained(
+            checkpoint,
+            subfolder="vae",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained(
+            checkpoint,
+            subfolder="scheduler",
+        )
+        dit = self.LongCatVideoTransformer3DModel.from_pretrained(
+            checkpoint,
+            subfolder="dit",
+            cp_split_hw=self.cp_split_hw,
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+            **overrides,
+        )
+        if self.options["use_distill"]:
+            dit.load_lora(
+                os.path.join(checkpoint, "lora", "cfg_step_lora.safetensors"),
+                "cfg_step_lora",
+            )
+            dit.enable_loras(["cfg_step_lora"])
+
+        self.pipeline = self.LongCatVideoPipeline(
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            vae=vae,
+            scheduler=scheduler,
+            dit=dit,
+        )
+        self.pipeline.to(self.device_index)
+
+    def _load_avatar_model(self, model):
+        avatar_patterns = list(AVATAR_COMMON_PATTERNS)
+        model_subfolder = (
+            "base_model_int8" if self.options["use_int8"] else "base_model"
+        )
+        avatar_patterns.append(f"{model_subfolder}/**")
+        checkpoint = self._resolve_checkpoint(model, avatar_patterns)
+
+        base_model = self.options.get("base_model")
+        if not base_model and os.path.isdir(normalize_model_source(model)):
+            sibling = os.path.join(
+                os.path.dirname(normalize_model_source(model)), "LongCat-Video"
+            )
+            if os.path.isdir(sibling):
+                base_model = sibling
+        base_model = base_model or BASE_MODEL_ID
+        if classify_model(str(base_model)) != MODEL_KIND_BASE:
+            raise ValueError("base_model must point to a LongCat-Video checkpoint")
+        base_checkpoint = self._resolve_checkpoint(base_model, AVATAR_BASE_PATTERNS)
+
+        dtype = self.torch.bfloat16
+        overrides = attention_overrides(self.options["attention_backend"])
+        tokenizer = self.AutoTokenizer.from_pretrained(
+            base_checkpoint,
+            subfolder="tokenizer",
+        )
+        text_encoder = self.UMT5EncoderModel.from_pretrained(
+            base_checkpoint,
+            subfolder="text_encoder",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        vae = self.AutoencoderKLWan.from_pretrained(
+            base_checkpoint,
+            subfolder="vae",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained(
+            checkpoint,
+            subfolder="scheduler",
+        )
+
+        if self.options["use_int8"]:
+            previous_dtype = self.torch.get_default_dtype()
+            self.torch.set_default_dtype(dtype)
+            try:
+                dit = self.load_quantized_dit(
+                    checkpoint,
+                    subfolder="base_model_int8",
+                    cp_split_hw=self.cp_split_hw,
+                    **overrides,
+                )
+            finally:
+                self.torch.set_default_dtype(previous_dtype)
+        else:
+            dit = self.LongCatVideoAvatarTransformer3DModel.from_pretrained(
+                checkpoint,
+                subfolder="base_model",
+                cp_split_hw=self.cp_split_hw,
+                torch_dtype=dtype,
+                low_cpu_mem_usage=True,
+                **overrides,
+            )
+
+        if self.options["use_distill"]:
+            dit.load_lora(
+                os.path.join(checkpoint, "lora", "dmd_lora.safetensors"),
+                "dmd",
+                multiplier=1.0,
+                lora_network_dim=128,
+                lora_network_alpha=64,
+            )
+            dit.enable_loras(["dmd"])
+
+        audio_checkpoint = os.path.join(checkpoint, "whisper-large-v3")
+        audio_encoder = self.get_audio_encoder(
+            audio_checkpoint,
+            MODEL_KIND_AVATAR + "-v1.5",
+        ).to(self.device_index)
+        audio_feature_extractor = self.get_audio_feature_extractor(
+            audio_checkpoint,
+            MODEL_KIND_AVATAR + "-v1.5",
+        )
+        self.pipeline = self.LongCatVideoAvatarPipeline(
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            vae=vae,
+            scheduler=scheduler,
+            dit=dit,
+            audio_encoder=audio_encoder,
+            audio_feature_extractor=audio_feature_extractor,
+            model_type="avatar-v1.5",
+        )
+        self.pipeline.to(self.device_index)
+
+    def _generate_base(self, request, params):
+        use_distill = self.options["use_distill"]
+        frames = normalize_num_frames(request.num_frames)
+        steps = (
+            16
+            if use_distill
+            else require_int(
+                request.step or 50,
+                "step",
+                minimum=1,
+                maximum=200,
+            )
+        )
+        guidance_scale = (
+            1.0
+            if use_distill
+            else require_float(
+                request.cfg_scale or 4.0,
+                "cfg_scale",
+                minimum=0.0,
+                maximum=30.0,
+            )
+        )
+        fps = require_int(request.fps or 15, "fps", minimum=1, maximum=60)
+        seed = request.seed if request.seed > 0 else 42
+        negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT
+        generator = self.torch.Generator(device=self.device_index).manual_seed(seed)
+
+        if request.start_image:
+            resolution = self._resolution(params)
+            image = self.load_image(request.start_image)
+            output = self.pipeline.generate_i2v(
+                image=image,
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                resolution=resolution,
+                num_frames=frames,
+                num_inference_steps=steps,
+                use_distill=use_distill,
+                guidance_scale=guidance_scale,
+                generator=generator,
+            )[0]
+        else:
+            width, height = validate_dimensions(request.width, request.height)
+            output = self.pipeline.generate_t2v(
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                height=height,
+                width=width,
+                num_frames=frames,
+                num_inference_steps=steps,
+                use_distill=use_distill,
+                guidance_scale=guidance_scale,
+                generator=generator,
+            )[0]
+
+        self._save_video(output, request.dst, fps)
+
+    def _generate_avatar(self, request, params, context):
+        if not request.audio:
+            raise ValueError("audio is required for LongCat-Video-Avatar-1.5")
+        if not os.path.isfile(request.audio):
+            raise ValueError("audio input is not a readable staged file")
+
+        use_distill = self.options["use_distill"]
+        steps = (
+            8
+            if use_distill
+            else require_int(
+                request.step or 50,
+                "step",
+                minimum=1,
+                maximum=200,
+            )
+        )
+        text_guidance = (
+            1.0
+            if use_distill
+            else require_float(
+                request.cfg_scale or 4.0,
+                "cfg_scale",
+                minimum=0.0,
+                maximum=30.0,
+            )
+        )
+        audio_guidance = (
+            1.0
+            if use_distill
+            else require_float(
+                params.get("audio_guidance_scale", 4.0),
+                "audio_guidance_scale",
+                minimum=0.0,
+                maximum=20.0,
+            )
+        )
+        seed = request.seed if request.seed > 0 else 42
+        generator = self.torch.Generator(device=self.device_index).manual_seed(seed)
+        negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT
+        resolution = self._resolution(params)
+
+        speech, sample_rate = self.librosa.load(request.audio, sr=16000, mono=True)
+        if speech.size == 0:
+            raise ValueError("audio contains no samples")
+        audio_duration = len(speech) / sample_rate
+        segments = self._avatar_segments(request, params, audio_duration)
+
+        segment_frames = 93
+        conditioning_frames = 13
+        avatar_fps = 25
+        generated_duration = (
+            segment_frames + (segments - 1) * (segment_frames - conditioning_frames)
+        ) / avatar_fps
+        pad_samples = max(
+            0, math.ceil((generated_duration - audio_duration) * sample_rate)
+        )
+        if pad_samples:
+            speech = self.np.pad(speech, (0, pad_samples))
+
+        full_audio_embedding = self.pipeline.get_audio_embedding(
+            speech,
+            fps=avatar_fps,
+            device=self.device_index,
+            sample_rate=sample_rate,
+            model_type="avatar-v1.5",
+        )
+        if not self.torch.isfinite(full_audio_embedding).all():
+            raise ValueError("audio encoder returned non-finite values")
+
+        indices = self.torch.arange(5) - 2
+
+        def audio_window(start_index):
+            centers = self.torch.arange(
+                start_index,
+                start_index + segment_frames,
+            ).unsqueeze(1) + indices.unsqueeze(0)
+            centers = self.torch.clamp(
+                centers,
+                min=0,
+                max=full_audio_embedding.shape[0] - 1,
+            )
+            return full_audio_embedding[centers][None, ...].to(self.device_index)
+
+        audio_start = 0
+        common = {
+            "prompt": request.prompt,
+            "negative_prompt": negative_prompt,
+            "num_frames": segment_frames,
+            "num_inference_steps": steps,
+            "text_guidance_scale": text_guidance,
+            "audio_guidance_scale": audio_guidance,
+            "output_type": "both",
+            "generator": generator,
+            "audio_emb": audio_window(audio_start),
+            "use_distill": use_distill,
+        }
+
+        if request.start_image:
+            output, latent = self.pipeline.generate_ai2v(
+                image=self.load_image(request.start_image),
+                resolution=resolution,
+                **common,
+            )
+        else:
+            width, height = validate_dimensions(request.width, request.height)
+            output, latent = self.pipeline.generate_at2v(
+                height=height,
+                width=width,
+                **common,
+            )
+
+        video = self._frames_to_pil(output[0])
+        width, height = video[0].size
+        current_video = video
+        reference_latent = latent[:, :, :1].clone()
+        all_frames = list(video)
+
+        for segment in range(1, segments):
+            if hasattr(context, "is_active") and not context.is_active():
+                raise RuntimeError("request was cancelled")
+            print(
+                f"Generating avatar segment {segment + 1}/{segments}", file=sys.stderr
+            )
+            audio_start += segment_frames - conditioning_frames
+            output, latent = self.pipeline.generate_avc(
+                video=current_video,
+                video_latent=latent,
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                height=height,
+                width=width,
+                num_frames=segment_frames,
+                num_cond_frames=conditioning_frames,
+                num_inference_steps=steps,
+                text_guidance_scale=text_guidance,
+                audio_guidance_scale=audio_guidance,
+                generator=generator,
+                output_type="both",
+                use_kv_cache=True,
+                offload_kv_cache=require_bool(
+                    params.get("offload_kv_cache", False),
+                    "offload_kv_cache",
+                ),
+                enhance_hf=not use_distill,
+                audio_emb=audio_window(audio_start),
+                ref_latent=reference_latent,
+                ref_img_index=require_int(
+                    params.get("ref_img_index", 10),
+                    "ref_img_index",
+                    minimum=-30,
+                    maximum=30,
+                ),
+                mask_frame_range=require_int(
+                    params.get("mask_frame_range", 3),
+                    "mask_frame_range",
+                    minimum=0,
+                    maximum=32,
+                ),
+                use_distill=use_distill,
+            )
+            current_video = self._frames_to_pil(output[0])
+            all_frames.extend(current_video[conditioning_frames:])
+
+        self._save_avatar_video(all_frames, request.audio, request.dst, avatar_fps)
+
+    def _avatar_segments(self, request, params, audio_duration):
+        if "num_segments" in params:
+            segments = require_int(
+                params["num_segments"],
+                "num_segments",
+                minimum=1,
+            )
+        elif request.num_frames > 0:
+            segments = avatar_segments_for_frames(request.num_frames)
+        else:
+            segments = avatar_segments_for_duration(audio_duration)
+
+        max_segments = self.options["max_segments"]
+        if segments > max_segments:
+            raise ValueError(
+                f"request needs {segments} avatar segments, but max_segments is {max_segments}; "
+                "trim the audio or raise the model's max_segments option"
+            )
+        return segments
+
+    def _resolution(self, params):
+        resolution = str(params.get("resolution", self.options["resolution"])).lower()
+        if resolution not in {"480p", "720p"}:
+            raise ValueError("resolution must be 480p or 720p")
+        return resolution
+
+    def _frames_to_pil(self, frames):
+        images = []
+        for frame in frames:
+            array = self.np.asarray(frame)
+            if self.np.issubdtype(array.dtype, self.np.floating):
+                array = self.np.clip(array, 0.0, 1.0) * 255
+            images.append(self.Image.fromarray(array.astype(self.np.uint8)))
+        return images
+
+    def _save_video(self, frames, path, fps):
+        writer = self.imageio.get_writer(
+            path,
+            format="FFMPEG",
+            mode="I",
+            fps=fps,
+            codec="libx264",
+            macro_block_size=1,
+            ffmpeg_params=[
+                "-crf",
+                "18",
+                "-pix_fmt",
+                "yuv420p",
+                "-movflags",
+                "+faststart",
+                "-f",
+                "mp4",
+            ],
+        )
+        try:
+            for frame in frames:
+                array = self.np.asarray(frame)
+                if self.np.issubdtype(array.dtype, self.np.floating):
+                    array = self.np.clip(array, 0.0, 1.0) * 255
+                writer.append_data(array.astype(self.np.uint8))
+        finally:
+            writer.close()
+
+    def _save_avatar_video(self, frames, audio_path, dst, fps):
+        output_dir = os.path.dirname(dst) or "."
+        handle, silent_path = tempfile.mkstemp(
+            prefix="longcat-silent-",
+            suffix=".mp4",
+            dir=output_dir,
+        )
+        os.close(handle)
+        try:
+            self._save_video(frames, silent_path, fps)
+            command = [
+                self.imageio_ffmpeg.get_ffmpeg_exe(),
+                "-y",
+                "-i",
+                silent_path,
+                "-i",
+                audio_path,
+                "-map",
+                "0:v:0",
+                "-map",
+                "1:a:0",
+                "-c:v",
+                "copy",
+                "-c:a",
+                "aac",
+                "-b:a",
+                "192k",
+                "-shortest",
+                "-movflags",
+                "+faststart",
+                "-f",
+                "mp4",
+                dst,
+            ]
+            subprocess.run(
+                command,
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+        except subprocess.CalledProcessError as err:
+            details = (err.stderr or "ffmpeg failed")[-2000:]
+            raise RuntimeError(f"failed to mux avatar audio: {details}") from err
+        finally:
+            try:
+                os.remove(silent_path)
+            except FileNotFoundError:
+                pass
+
+    def _release_model(self):
+        self.pipeline = None
+        self.model_kind = None
+        gc.collect()
+        if hasattr(self, "torch") and self.torch.cuda.is_available():
+            self.torch.cuda.empty_cache()
+            self.torch.cuda.ipc_collect()
+
+    @staticmethod
+    def _fail(context, code, message):
+        if context is not None:
+            context.set_code(code)
+            context.set_details(message)
+        return backend_pb2.Result(message=message, success=False)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_message_length", 64 * 1024 * 1024),
+            ("grpc.max_send_message_length", 64 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 64 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"LongCat Video backend listening on {address}", file=sys.stderr)
+
+    def stop_server(signum, frame):
+        del signum, frame
+        server.stop(0)
+
+    signal.signal(signal.SIGINT, stop_server)
+    signal.signal(signal.SIGTERM, stop_server)
+    server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the LongCat Video gRPC backend")
+    parser.add_argument(
+        "--addr",
+        default="localhost:50051",
+        help="address on which to serve the backend",
+    )
+    arguments = parser.parse_args()
+    serve(arguments.addr)
--- a/backend/python/longcat-video/install.sh
+++ b/backend/python/longcat-video/install.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+PYTHON_VERSION="3.12"
+PYTHON_PATCH="12"
+PY_STANDALONE_TAG="20251120"
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+installRequirements
--- a/backend/python/longcat-video/longcat_utils.py
+++ b/backend/python/longcat-video/longcat_utils.py
@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: MIT
+
+import json
+import math
+import os
+from urllib.parse import urlparse
+
+
+BASE_MODEL_ID = "meituan-longcat/LongCat-Video"
+AVATAR_MODEL_ID = "meituan-longcat/LongCat-Video-Avatar-1.5"
+MODEL_KIND_BASE = "base"
+MODEL_KIND_AVATAR = "avatar"
+
+ATTENTION_OVERRIDES = {
+    "auto": {},
+    "sdpa": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": False,
+        "enable_xformers": False,
+    },
+    "flash2": {
+        "enable_flashattn2": True,
+        "enable_flashattn3": False,
+        "enable_xformers": False,
+    },
+    "flash3": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": True,
+        "enable_xformers": False,
+    },
+    "xformers": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": False,
+        "enable_xformers": True,
+    },
+}
+
+
+def parse_options(values):
+    options = {}
+    for raw in values:
+        if ":" not in raw:
+            options[raw.strip()] = True
+            continue
+        key, value = raw.split(":", 1)
+        key = key.strip()
+        value = value.strip()
+        if not key:
+            continue
+        lower = value.lower()
+        if lower in {"true", "false"}:
+            options[key] = lower == "true"
+            continue
+        try:
+            options[key] = int(value)
+            continue
+        except ValueError:
+            pass
+        try:
+            options[key] = float(value)
+            continue
+        except ValueError:
+            pass
+        options[key] = value
+    return options
+
+
+def require_bool(value, name):
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.lower() in {"true", "false"}:
+        return value.lower() == "true"
+    raise ValueError(f"{name} must be true or false")
+
+
+def require_int(value, name, minimum=None, maximum=None):
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as err:
+        raise ValueError(f"{name} must be an integer") from err
+    if minimum is not None and parsed < minimum:
+        raise ValueError(f"{name} must be at least {minimum}")
+    if maximum is not None and parsed > maximum:
+        raise ValueError(f"{name} must be at most {maximum}")
+    return parsed
+
+
+def require_float(value, name, minimum=None, maximum=None):
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError) as err:
+        raise ValueError(f"{name} must be a number") from err
+    if minimum is not None and parsed < minimum:
+        raise ValueError(f"{name} must be at least {minimum}")
+    if maximum is not None and parsed > maximum:
+        raise ValueError(f"{name} must be at most {maximum}")
+    return parsed
+
+
+def attention_overrides(name):
+    try:
+        return dict(ATTENTION_OVERRIDES[name])
+    except KeyError as err:
+        choices = ", ".join(ATTENTION_OVERRIDES)
+        raise ValueError(f"attention_backend must be one of: {choices}") from err
+
+
+def _model_name_from_directory(path):
+    for filename in ("model_index.json", "config.json"):
+        config_path = os.path.join(path, filename)
+        try:
+            with open(config_path, "r", encoding="utf-8") as config_file:
+                model_name = json.load(config_file).get("model_name", "")
+        except (FileNotFoundError, OSError, ValueError, TypeError):
+            continue
+        if model_name:
+            return model_name
+    return ""
+
+
+def normalize_model_source(model):
+    value = model.rstrip("/")
+    for prefix in ("huggingface://", "hf://"):
+        if value.startswith(prefix):
+            return value[len(prefix) :]
+    parsed = urlparse(value)
+    if parsed.scheme in {"http", "https"} and parsed.netloc.lower() == "huggingface.co":
+        parts = [part for part in parsed.path.split("/") if part]
+        if len(parts) >= 2:
+            return "/".join(parts[:2])
+    return value
+
+
+def classify_model(model):
+    if not model:
+        return None
+    normalized = normalize_model_source(model)
+    if os.path.isdir(normalized):
+        name = _model_name_from_directory(normalized).lower()
+        if name == "longcat-video":
+            return MODEL_KIND_BASE
+        if name == "longcat-video-avatar-1.5":
+            return MODEL_KIND_AVATAR
+        return None
+
+    normalized = normalized.lower()
+    if normalized == BASE_MODEL_ID.lower():
+        return MODEL_KIND_BASE
+    if normalized == AVATAR_MODEL_ID.lower():
+        return MODEL_KIND_AVATAR
+    return None
+
+
+def normalize_num_frames(value, default=93):
+    frames = default if not value or value < 1 else value
+    return max(1, ((frames - 1) // 4) * 4 + 1)
+
+
+def avatar_segments_for_frames(frames):
+    if not frames or frames <= 93:
+        return 1
+    return 1 + math.ceil((frames - 93) / 80)
+
+
+def avatar_segments_for_duration(duration_seconds, fps=25):
+    if duration_seconds <= 0:
+        return 1
+    return avatar_segments_for_frames(math.ceil(duration_seconds * fps))
+
+
+def validate_dimensions(width, height):
+    width = width or 832
+    height = height or 480
+    if width < 256 or height < 256:
+        raise ValueError("width and height must each be at least 256")
+    if width > 1280 or height > 768:
+        raise ValueError("width and height must not exceed 1280x768")
+    if width % 16 != 0 or height % 16 != 0:
+        raise ValueError("width and height must be divisible by 16")
+    if width * height > 1280 * 768:
+        raise ValueError("requested video dimensions exceed the 1280x768 pixel limit")
+    return width, height
--- a/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch
+++ b/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch
@@ -0,0 +1,75 @@
+diff --git a/longcat_video/modules/attention.py b/longcat_video/modules/attention.py
+index bb5630f..9b9f3cc 100644
+--- a/longcat_video/modules/attention.py
+++ b/longcat_video/modules/attention.py
+@@ -2,6 +2,7 @@ from typing import List, Optional
+
+ import torch
+ import torch.nn as nn
+import torch.nn.functional as F
+
+ from einops import rearrange
+
+@@ -100,7 +101,8 @@ class Attention(nn.Module):
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,)
+             x = rearrange(x, "B M H K -> B H M K")
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
+            # Keep a dependency-free path for systems without optional kernels.
+            x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
+
+         return x
+
+@@ -245,8 +247,22 @@ class MultiHeadCrossAttention(nn.Module):
+             attn_bias = xformers.ops.fmha.attn_bias.BlockDiagonalMask.from_seqlens([N] * B, kv_seqlen)
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
+-
+            # Preserve the variable-length block boundaries without materializing
+            # a dense attention mask.
+            blocks = []
+            offset = 0
+            for batch_index, key_count in enumerate(kv_seqlen):
+                query = q[0][batch_index * N:(batch_index + 1) * N]
+                key = k[0][offset:offset + key_count]
+                value = v[0][offset:offset + key_count]
+                output = F.scaled_dot_product_attention(
+                    query.transpose(0, 1),
+                    key.transpose(0, 1),
+                    value.transpose(0, 1),
+                )
+                blocks.append(output.transpose(0, 1))
+                offset += key_count
+            x = torch.cat(blocks, dim=0)
+
+         x = x.view(B, -1, C)
+         x = self.proj(x)
+diff --git a/longcat_video/modules/avatar/attention.py b/longcat_video/modules/avatar/attention.py
+index a169a7a..df9a469 100644
+--- a/longcat_video/modules/avatar/attention.py
+++ b/longcat_video/modules/avatar/attention.py
+@@ -2,6 +2,7 @@ from typing import List, Optional
+
+ import torch
+ import torch.nn as nn
+import torch.nn.functional as F
+
+ from einops import rearrange
+
+@@ -111,7 +112,8 @@ class Attention(nn.Module):
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,)
+             x = rearrange(x, "B M H K -> B H M K")
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
+            # Keep a dependency-free path for systems without optional kernels.
+            x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
+
+         return x
+
+@@ -429,2 +431,5 @@ class SingleStreamAttention(nn.Module):
+        else:
+            # This branch uses the native PyTorch kernel when optional kernels are off.
+            x = F.scaled_dot_product_attention(q, encoder_k, encoder_v, scale=self.scale)
+
+         # linear transform
--- a/backend/python/longcat-video/requirements-after.txt
+++ b/backend/python/longcat-video/requirements-after.txt
@@ -0,0 +1 @@
+accelerate
--- a/backend/python/longcat-video/requirements-cpu.txt
+++ b/backend/python/longcat-video/requirements-cpu.txt
@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cpu
+torch==2.12.1
+torchvision==0.27.1
--- a/backend/python/longcat-video/requirements-cublas12.txt
+++ b/backend/python/longcat-video/requirements-cublas12.txt
@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu126
+torch==2.12.1
+torchvision==0.27.1
--- a/backend/python/longcat-video/requirements-cublas13.txt
+++ b/backend/python/longcat-video/requirements-cublas13.txt
@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu130
+torch==2.12.1
+torchvision==0.27.1
--- a/backend/python/longcat-video/requirements-l4t13.txt
+++ b/backend/python/longcat-video/requirements-l4t13.txt
@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu130
+torch==2.12.1
+torchvision==0.27.1
--- a/backend/python/longcat-video/requirements.txt
+++ b/backend/python/longcat-video/requirements.txt
@@ -0,0 +1,23 @@
+certifi
+diffusers==0.35.1
+einops==0.8.0
+ftfy==6.2.0
+grpcio==1.76.0
+huggingface-hub>=0.23,<1.0
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+librosa==0.11.0
+loguru==0.7.2
+numpy==1.26.4
+packaging
+pillow
+protobuf
+pyloudnorm==0.1.1
+regex
+safetensors
+scipy==1.15.3
+sentencepiece
+soundfile==0.13.1
+soxr==0.5.0.post1
+tqdm
+transformers==4.41.0
--- a/backend/python/longcat-video/run.sh
+++ b/backend/python/longcat-video/run.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+startBackend "$@"
--- a/backend/python/longcat-video/test.py
+++ b/backend/python/longcat-video/test.py
@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: MIT
+
+import importlib.util
+import json
+import os
+import sys
+import tempfile
+import unittest
+
+BACKEND_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, BACKEND_DIR)
+
+# longcat-video is a backend directory, not an importable Python package name.
+from longcat_utils import (  # noqa: E402
+    MODEL_KIND_AVATAR,
+    MODEL_KIND_BASE,
+    attention_overrides,
+    avatar_segments_for_duration,
+    avatar_segments_for_frames,
+    classify_model,
+    normalize_model_source,
+    normalize_num_frames,
+    parse_options,
+    validate_dimensions,
+)
+
+
+SOURCE_DIR = os.path.join(BACKEND_DIR, "sources", "LongCat-Video")
+try:
+    import torch
+
+    sys.path.insert(0, SOURCE_DIR)
+    ATTENTION_TESTS_AVAILABLE = (
+        os.path.isdir(SOURCE_DIR) and importlib.util.find_spec("triton") is not None
+    )
+except ImportError:
+    torch = None
+    ATTENTION_TESTS_AVAILABLE = False
+
+AVATAR_ATTENTION_TESTS_AVAILABLE = ATTENTION_TESTS_AVAILABLE and all(
+    importlib.util.find_spec(module) is not None
+    for module in ("pyloudnorm", "scipy", "torchvision")
+)
+
+
+class LongCatUtilsTest(unittest.TestCase):
+    def test_parse_options_preserves_colons_and_coerces_scalars(self):
+        options = parse_options(
+            [
+                "use_distill:true",
+                "max_segments:4",
+                "audio_guidance_scale:3.5",
+                "source:https://example.com/model",
+                "flag",
+            ]
+        )
+
+        self.assertEqual(options["use_distill"], True)
+        self.assertEqual(options["max_segments"], 4)
+        self.assertEqual(options["audio_guidance_scale"], 3.5)
+        self.assertEqual(options["source"], "https://example.com/model")
+        self.assertEqual(options["flag"], True)
+
+    def test_classify_model_accepts_only_supported_longcat_models(self):
+        cases = {
+            "meituan-longcat/LongCat-Video": MODEL_KIND_BASE,
+            "https://huggingface.co/meituan-longcat/LongCat-Video": MODEL_KIND_BASE,
+            "hf://meituan-longcat/LongCat-Video-Avatar-1.5": MODEL_KIND_AVATAR,
+            "other-org/LongCat-Video": None,
+            "meituan-longcat/LongCat-Video-Avatar": None,
+            "some-org/unrelated-model": None,
+        }
+
+        for model, expected in cases.items():
+            with self.subTest(model=model):
+                self.assertEqual(classify_model(model), expected)
+
+    def test_classify_model_reads_local_checkpoint_metadata(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with open(
+                os.path.join(directory, "model_index.json"),
+                "w",
+                encoding="utf-8",
+            ) as config_file:
+                json.dump({"model_name": "LongCat-Video-Avatar-1.5"}, config_file)
+
+            self.assertEqual(classify_model(directory), MODEL_KIND_AVATAR)
+
+    def test_normalize_model_source_handles_huggingface_uri_forms(self):
+        self.assertEqual(
+            normalize_model_source(
+                "https://huggingface.co/meituan-longcat/LongCat-Video/tree/main"
+            ),
+            "meituan-longcat/LongCat-Video",
+        )
+        self.assertEqual(
+            normalize_model_source("huggingface://meituan-longcat/LongCat-Video"),
+            "meituan-longcat/LongCat-Video",
+        )
+
+    def test_frame_and_segment_rounding_matches_longcat_temporal_shape(self):
+        self.assertEqual(normalize_num_frames(94), 93)
+        self.assertEqual(normalize_num_frames(0), 93)
+        self.assertEqual(avatar_segments_for_frames(93), 1)
+        self.assertEqual(avatar_segments_for_frames(94), 2)
+        self.assertEqual(avatar_segments_for_frames(173), 2)
+        self.assertEqual(avatar_segments_for_frames(174), 3)
+        self.assertEqual(avatar_segments_for_duration(10.0), 3)
+
+    def test_dimensions_are_bounded_and_aligned(self):
+        self.assertEqual(validate_dimensions(0, 0), (832, 480))
+        self.assertEqual(validate_dimensions(512, 512), (512, 512))
+        with self.assertRaisesRegex(ValueError, "divisible by 16"):
+            validate_dimensions(513, 512)
+        with self.assertRaisesRegex(ValueError, "must not exceed"):
+            validate_dimensions(1920, 1080)
+
+    def test_attention_backend_validation(self):
+        self.assertEqual(
+            attention_overrides("sdpa"),
+            {
+                "enable_flashattn2": False,
+                "enable_flashattn3": False,
+                "enable_xformers": False,
+            },
+        )
+        with self.assertRaisesRegex(ValueError, "attention_backend"):
+            attention_overrides("unknown")
+
+
+@unittest.skipUnless(
+    ATTENTION_TESTS_AVAILABLE,
+    "patched LongCat source and torch are required for attention tests",
+)
+class SDPAFallbackTest(unittest.TestCase):
+    def test_base_self_attention_matches_reference(self):
+        from longcat_video.modules.attention import Attention
+
+        dim, heads, sequence = 64, 4, 32
+        attention = Attention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+            enable_bsa=False,
+        ).float()
+        query = torch.randn(2, heads, sequence, dim // heads)
+        key = torch.randn_like(query)
+        value = torch.randn_like(query)
+
+        output = attention._process_attn(query, key, value, shape=(1, 1, sequence))
+        reference = (
+            torch.softmax(
+                (query @ key.transpose(-1, -2)) * attention.scale,
+                dim=-1,
+            )
+            @ value
+        )
+
+        self.assertLess((output - reference).abs().max().item(), 1e-4)
+
+    @unittest.skipUnless(
+        AVATAR_ATTENTION_TESTS_AVAILABLE,
+        "avatar audio dependencies are required for the avatar attention test",
+    )
+    def test_avatar_self_attention_matches_reference(self):
+        from longcat_video.modules.avatar.attention import Attention
+
+        dim, heads, sequence = 64, 4, 16
+        attention = Attention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+        ).float()
+        query = torch.randn(1, heads, sequence, dim // heads)
+        key = torch.randn_like(query)
+        value = torch.randn_like(query)
+
+        output = attention._process_attn(query, key, value, shape=(1, 1, sequence))
+        reference = (
+            torch.softmax(
+                (query @ key.transpose(-1, -2)) * attention.scale,
+                dim=-1,
+            )
+            @ value
+        )
+
+        self.assertLess((output - reference).abs().max().item(), 1e-4)
+
+    def test_base_cross_attention_remains_block_diagonal(self):
+        from longcat_video.modules.attention import MultiHeadCrossAttention
+
+        dim, heads = 64, 4
+        attention = MultiHeadCrossAttention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+        ).float()
+        query = torch.randn(2, 8, dim)
+        key_lengths = [5, 7]
+        condition = torch.randn(1, sum(key_lengths), dim)
+
+        first = attention._process_cross_attn(query, condition, key_lengths)
+        changed = condition.clone()
+        changed[:, key_lengths[0] :] = torch.randn_like(changed[:, key_lengths[0] :])
+        second = attention._process_cross_attn(query, changed, key_lengths)
+
+        self.assertLess((first[0] - second[0]).abs().max().item(), 1e-5)
+        self.assertGreater((first[1] - second[1]).abs().max().item(), 1e-3)
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/backend/python/longcat-video/test.sh
+++ b/backend/python/longcat-video/test.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+runUnittests
--- a/core/backend/video.go
+++ b/core/backend/video.go
@@ -1,21 +1,37 @@
 package backend

 import (
+	"maps"
 	"time"

 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/trace"
-
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	model "github.com/mudler/LocalAI/pkg/model"
 )

-func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, endImage, dst string, numFrames, fps, seed int32, cfgScale float32, step int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
+// VideoGenerationOptions is the backend-neutral request passed to video generators.
+// Media fields contain staged local paths by the time they reach this layer.
+type VideoGenerationOptions struct {
+	Height         int32
+	Width          int32
+	Prompt         string
+	NegativePrompt string
+	StartImage     string
+	EndImage       string
+	Audio          string
+	Destination    string
+	NumFrames      int32
+	FPS            int32
+	Seed           int32
+	CFGScale       float32
+	Step           int32
+	Params         map[string]string
+}

+func VideoGeneration(options VideoGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
 	opts := ModelOptions(modelConfig, appConfig)
-	inferenceModel, err := loader.Load(
-		opts...,
-	)
+	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
 		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
@@ -25,19 +41,22 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 		_, err := inferenceModel.GenerateVideo(
 			appConfig.Context,
 			&proto.GenerateVideoRequest{
-				Height:         height,
-				Width:          width,
-				Prompt:         prompt,
-				NegativePrompt: negativePrompt,
-				StartImage:     startImage,
-				EndImage:       endImage,
-				NumFrames:      numFrames,
-				Fps:            fps,
-				Seed:           seed,
-				CfgScale:       cfgScale,
-				Step:           step,
-				Dst:            dst,
-			})
+				Height:         options.Height,
+				Width:          options.Width,
+				Prompt:         options.Prompt,
+				NegativePrompt: options.NegativePrompt,
+				StartImage:     options.StartImage,
+				EndImage:       options.EndImage,
+				Audio:          options.Audio,
+				NumFrames:      options.NumFrames,
+				Fps:            options.FPS,
+				Seed:           options.Seed,
+				CfgScale:       options.CFGScale,
+				Step:           options.Step,
+				Dst:            options.Destination,
+				Params:         maps.Clone(options.Params),
+			},
+		)
 		return err
 	}

@@ -45,15 +64,18 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes)

 		traceData := map[string]any{
-			"prompt":          prompt,
-			"negative_prompt": negativePrompt,
-			"height":          height,
-			"width":           width,
-			"num_frames":      numFrames,
-			"fps":             fps,
-			"seed":            seed,
-			"cfg_scale":       cfgScale,
-			"step":            step,
+			"prompt":          options.Prompt,
+			"negative_prompt": options.NegativePrompt,
+			"height":          options.Height,
+			"width":           options.Width,
+			"num_frames":      options.NumFrames,
+			"fps":             options.FPS,
+			"seed":            options.Seed,
+			"cfg_scale":       options.CFGScale,
+			"step":            options.Step,
+			"has_start_image": options.StartImage != "",
+			"has_end_image":   options.EndImage != "",
+			"has_audio":       options.Audio != "",
 		}

 		startTime := time.Now()
@@ -73,7 +95,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 				Type:      trace.BackendTraceVideoGeneration,
 				ModelName: modelConfig.Name,
 				Backend:   modelConfig.Backend,
-				Summary:   trace.TruncateString(prompt, 200),
+				Summary:   trace.TruncateString(options.Prompt, 200),
 				Error:     errStr,
 				Data:      traceData,
 			})
--- a/core/config/backend_capabilities.go
+++ b/core/config/backend_capabilities.go
@@ -120,7 +120,7 @@ var UsecaseInfoMap = map[string]UsecaseInfo{
 	UsecaseVideo: {
 		Flag:        FLAG_VIDEO,
 		GRPCMethod:  MethodGenerateVideo,
-		Description: "Video generation via the GenerateVideo RPC.",
+		Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.",
 	},
 	UsecaseTranscript: {
 		Flag:        FLAG_TRANSCRIPT,
@@ -303,6 +303,14 @@ var BackendCapabilities = map[string]BackendCapability{
 		DefaultUsecases:  []string{UsecaseImage},
 		Description:      "HuggingFace diffusers — Stable Diffusion, Flux, video generation",
 	},
+	"longcat-video": {
+		GRPCMethods:      []GRPCMethod{MethodGenerateVideo},
+		PossibleUsecases: []string{UsecaseVideo},
+		DefaultUsecases:  []string{UsecaseVideo},
+		AcceptsImages:    true,
+		AcceptsAudios:    true,
+		Description:      "LongCat-Video — text, image, and audio-conditioned avatar video generation on NVIDIA CUDA",
+	},
 	"stablediffusion": {
 		GRPCMethods:      []GRPCMethod{MethodGenerateImage},
 		PossibleUsecases: []string{UsecaseImage},
--- a/core/config/meta/constants.go
+++ b/core/config/meta/constants.go
@@ -79,6 +79,14 @@ var UsecaseOptions = []FieldOption{
 	{Value: "video", Label: "Video"},
 }

+// ModalityOptions enumerates the values accepted by known modality fields.
+var ModalityOptions = []FieldOption{
+	{Value: "text", Label: "Text"},
+	{Value: "image", Label: "Image"},
+	{Value: "audio", Label: "Audio"},
+	{Value: "video", Label: "Video"},
+}
+
 var DiffusersSchedulerOptions = []FieldOption{
 	{Value: "ddim", Label: "DDIM"},
 	{Value: "ddpm", Label: "DDPM"},
--- a/core/config/meta/registry.go
+++ b/core/config/meta/registry.go
@@ -66,6 +66,22 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Options:     UsecaseOptions,
 			Order:       6,
 		},
+		"known_input_modalities": {
+			Section:     "general",
+			Label:       "Known Input Modalities",
+			Description: "Explicit input types this model accepts when use cases alone are not specific enough",
+			Component:   "string-list",
+			Options:     ModalityOptions,
+			Order:       7,
+		},
+		"known_output_modalities": {
+			Section:     "general",
+			Label:       "Known Output Modalities",
+			Description: "Explicit output types this model produces when use cases alone are not specific enough",
+			Component:   "string-list",
+			Options:     ModalityOptions,
+			Order:       8,
+		},

 		// --- LLM ---
 		"context_size": {
--- a/core/config/model_capabilities.go
+++ b/core/config/model_capabilities.go
@@ -1,5 +1,7 @@
 package config

+import "slices"
+
 // This file is the single source of truth for deriving a model's user-facing
 // capabilities and input/output modalities from its ModelConfig. Both the
 // OpenAI-compatible /v1/models/capabilities endpoint and the Ollama-compatible
@@ -7,14 +9,47 @@ package config
 // across clients. Keep the detection heuristics here rather than duplicating
 // them per endpoint.

+// Canonical model modality values used by config declarations and discovery APIs.
+const (
+	ModalityText  = "text"
+	ModalityImage = "image"
+	ModalityAudio = "audio"
+	ModalityVideo = "video"
+)
+
+var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo}
+
+func declaredModalities(modalities []string) map[string]bool {
+	declared := make(map[string]bool, len(modalities))
+	for _, modality := range modalities {
+		if slices.Contains(modalityOrder, modality) {
+			declared[modality] = true
+		}
+	}
+	return declared
+}
+
+func orderedModalities(modalities map[string]bool) []string {
+	result := make([]string, 0, len(modalityOrder))
+	for _, modality := range modalityOrder {
+		if modalities[modality] {
+			result = append(result, modality)
+		}
+	}
+	return result
+}
+
 // VisionSupported reports whether the model can accept image inputs.
 //
 // We deliberately avoid HasUsecases(FLAG_VISION): GuessUsecases has no
 // FLAG_VISION branch and reports true for any chat model, so it would paint
 // vision onto text-only models. Instead we look for explicit signals: the
-// declared KnownUsecases bit, a multimodal projector, or a template/backend
-// multimodal marker.
+// declared input modality or KnownUsecases bit, a multimodal projector, or a
+// template/backend multimodal marker.
 func (c *ModelConfig) VisionSupported() bool {
+	if slices.Contains(c.KnownInputModalities, ModalityImage) {
+		return true
+	}
 	if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION {
 		return true
 	}
@@ -68,20 +103,21 @@ func (c *ModelConfig) ThinkingSupported() bool {
 }

 // AudioInputSupported reports whether a chat/generation model accepts audio as
-// input (e.g. vLLM omni models). The signal is the vLLM per-prompt audio limit;
-// there is no FLAG_* for "chat model that hears audio", which is exactly why a
-// plain usecase list can't express it. Transcription models are handled
-// separately in InputModalities via FLAG_TRANSCRIPT.
+// input. Model configs can declare this directly; vLLM-family configs can also
+// signal it through the per-prompt audio limit. Transcription models are
+// handled separately in InputModalities via FLAG_TRANSCRIPT.
 func (c *ModelConfig) AudioInputSupported() bool {
-	return c.LimitMMPerPrompt.LimitAudioPerPrompt > 0
+	return slices.Contains(c.KnownInputModalities, ModalityAudio) ||
+		c.LimitMMPerPrompt.LimitAudioPerPrompt > 0
 }

 // VideoInputSupported reports whether a chat/generation model accepts video as
-// input. The signal is the vLLM per-prompt video limit. Note this is distinct
-// from FLAG_VIDEO, which denotes video *generation* (diffusers) — an output
-// modality, not an input one.
+// input. Model configs can declare this directly; vLLM-family configs can also
+// signal it through the per-prompt video limit. This is distinct from
+// FLAG_VIDEO, which denotes video generation — an output modality.
 func (c *ModelConfig) VideoInputSupported() bool {
-	return c.LimitMMPerPrompt.LimitVideoPerPrompt > 0
+	return slices.Contains(c.KnownInputModalities, ModalityVideo) ||
+		c.LimitMMPerPrompt.LimitVideoPerPrompt > 0
 }

 // Capabilities returns the ordered list of capability strings the model
@@ -135,6 +171,7 @@ func (c *ModelConfig) Capabilities() []string {
 // attachment router consults to decide whether an image/audio/video file can be
 // handed to the active model directly.
 func (c *ModelConfig) InputModalities() []string {
+	modalities := declaredModalities(c.KnownInputModalities)
 	imageGen := c.HasUsecases(FLAG_IMAGE)
 	videoGen := c.HasUsecases(FLAG_VIDEO)
 	chatish := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION)
@@ -154,25 +191,17 @@ func (c *ModelConfig) InputModalities() []string {

 	videoIn := c.VideoInputSupported()

-	var mods []string
-	if textIn {
-		mods = append(mods, "text")
-	}
-	if imageIn {
-		mods = append(mods, "image")
-	}
-	if audioIn {
-		mods = append(mods, "audio")
-	}
-	if videoIn {
-		mods = append(mods, "video")
-	}
-	return mods
+	modalities[ModalityText] = modalities[ModalityText] || textIn
+	modalities[ModalityImage] = modalities[ModalityImage] || imageIn
+	modalities[ModalityAudio] = modalities[ModalityAudio] || audioIn
+	modalities[ModalityVideo] = modalities[ModalityVideo] || videoIn
+	return orderedModalities(modalities)
 }

 // OutputModalities returns the set of modalities (text, image, audio, video)
 // the model produces, ordered text→image→audio→video.
 func (c *ModelConfig) OutputModalities() []string {
+	modalities := declaredModalities(c.KnownOutputModalities)
 	textOut := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) || c.HasUsecases(FLAG_EDIT) ||
 		c.HasUsecases(FLAG_TRANSCRIPT)
 	imageOut := c.HasUsecases(FLAG_IMAGE)
@@ -180,18 +209,9 @@ func (c *ModelConfig) OutputModalities() []string {
 		c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO)
 	videoOut := c.HasUsecases(FLAG_VIDEO)

-	var mods []string
-	if textOut {
-		mods = append(mods, "text")
-	}
-	if imageOut {
-		mods = append(mods, "image")
-	}
-	if audioOut {
-		mods = append(mods, "audio")
-	}
-	if videoOut {
-		mods = append(mods, "video")
-	}
-	return mods
+	modalities[ModalityText] = modalities[ModalityText] || textOut
+	modalities[ModalityImage] = modalities[ModalityImage] || imageOut
+	modalities[ModalityAudio] = modalities[ModalityAudio] || audioOut
+	modalities[ModalityVideo] = modalities[ModalityVideo] || videoOut
+	return orderedModalities(modalities)
 }
--- a/core/config/model_capabilities_test.go
+++ b/core/config/model_capabilities_test.go
@@ -21,6 +21,14 @@ var _ = Describe("Model capabilities derivation", func() {
 			Expect(cfg.VisionSupported()).To(BeTrue())
 		})

+		It("is true when image input is declared explicitly", func() {
+			cfg := &ModelConfig{
+				KnownUsecases:        usecaseBits(FLAG_CHAT),
+				KnownInputModalities: []string{ModalityText, ModalityImage},
+			}
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
 		It("is true when an mmproj projector is set", func() {
 			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"}
 			cfg.MMProj = "mmproj.gguf" // promoted field from the embedded options struct
@@ -37,6 +45,14 @@ var _ = Describe("Model capabilities derivation", func() {
 	})

 	Describe("AudioInputSupported / VideoInputSupported", func() {
+		It("honors explicit model modality declarations", func() {
+			cfg := &ModelConfig{
+				KnownInputModalities: []string{ModalityAudio, ModalityVideo},
+			}
+			Expect(cfg.AudioInputSupported()).To(BeTrue())
+			Expect(cfg.VideoInputSupported()).To(BeTrue())
+		})
+
 		It("detects vLLM omni audio input via limit_mm_per_prompt", func() {
 			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"}
 			cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1
@@ -93,6 +109,17 @@ var _ = Describe("Model capabilities derivation", func() {
 			Expect(cfg.OutputModalities()).To(Equal([]string{"image"}))
 		})

+		It("conditioned video uses declared modalities without backend-specific inference", func() {
+			cfg := &ModelConfig{
+				KnownUsecases:         usecaseBits(FLAG_VIDEO),
+				KnownInputModalities:  []string{ModalityAudio, ModalityImage, ModalityText, ModalityAudio, "unknown"},
+				KnownOutputModalities: []string{ModalityVideo},
+			}
+			Expect(cfg.Capabilities()).To(Equal([]string{UsecaseVideo}))
+			Expect(cfg.InputModalities()).To(Equal([]string{ModalityText, ModalityImage, ModalityAudio}))
+			Expect(cfg.OutputModalities()).To(Equal([]string{ModalityVideo}))
+		})
+
 		It("a TTS model reads text and writes audio", func() {
 			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "piper"}
 			Expect(cfg.Capabilities()).To(ContainElement(UsecaseTTS))
--- a/core/config/model_config.go
+++ b/core/config/model_config.go
@@ -52,7 +52,11 @@ type ModelConfig struct {
 	TemplateConfig      TemplateConfig      `yaml:"template,omitempty" json:"template,omitempty"`
 	KnownUsecaseStrings []string            `yaml:"known_usecases,omitempty" json:"known_usecases,omitempty"`
 	KnownUsecases       *ModelConfigUsecase `yaml:"-" json:"-"`
-	Pipeline            Pipeline            `yaml:"pipeline,omitempty" json:"pipeline,omitempty"`
+	// KnownInputModalities and KnownOutputModalities describe model-specific I/O
+	// that usecases alone cannot express, such as image- or audio-conditioned video.
+	KnownInputModalities  []string `yaml:"known_input_modalities,omitempty" json:"known_input_modalities,omitempty"`
+	KnownOutputModalities []string `yaml:"known_output_modalities,omitempty" json:"known_output_modalities,omitempty"`
+	Pipeline              Pipeline `yaml:"pipeline,omitempty" json:"pipeline,omitempty"`

 	PromptStrings, InputStrings                []string       `yaml:"-" json:"-"`
 	InputToken                                 [][]int        `yaml:"-" json:"-"`
--- a/core/gallery/importers/importers.go
+++ b/core/gallery/importers/importers.go
@@ -33,7 +33,7 @@ var ErrAmbiguousImport = errors.New("importer: ambiguous — specify preferences
 // pipeline_tag values.
 type AmbiguousImportError struct {
 	// Modality is the importer modality key ("text", "asr", "tts", "image",
-	// "embeddings", "reranker", "detection"). Pre-mapped from the HF
+	// "video", "embeddings", "reranker", "detection"). Pre-mapped from the HF
 	// pipeline_tag so the UI doesn't have to.
 	Modality string
 	// Candidates is the list of backend names whose Modality() matches — a
@@ -144,6 +144,9 @@ var defaultImporters = []Importer{
 	// Image/Video (Batch 3)
 	&StableDiffusionGGMLImporter{},
 	&ACEStepImporter{},
+	// LongCat repositories carry generic Diffusers metadata, so this exact
+	// owner/repo matcher must run before DiffuserImporter.
+	&LongCatVideoImporter{},
 	// Text LLM (Batch 4) — VLLMOmniImporter must stay ahead of
 	// VLLMImporter so Qwen Omni repos (which also carry tokenizer
 	// files) route to vllm-omni rather than plain vllm.
@@ -204,7 +207,7 @@ type Importer interface {
 	// /backends/known to populate the import form dropdown.
 	Name() string
 	// Modality is the backend's primary modality ("text", "asr", "tts",
-	// "image", "embeddings", "reranker", "detection", "vad"). Used for
+	// "image", "video", "embeddings", "reranker", "detection", "vad"). Used for
 	// grouping in the UI.
 	Modality() string
 	// AutoDetects is true when Match() can fire without an explicit
--- a/core/gallery/importers/importers_test.go
+++ b/core/gallery/importers/importers_test.go
@@ -15,6 +15,16 @@ import (
 var _ = Describe("DiscoverModelConfig", func() {

 	Context("With only a repository URI", func() {
+		It("should discover LongCat Avatar before the generic Diffusers importer", func() {
+			uri := "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5"
+
+			modelConfig, err := importers.DiscoverModelConfig(uri, json.RawMessage(`{}`))
+
+			Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:\n    - video"))
+		})
+
 		It("should discover and import using LlamaCPPImporter", func() {
 			uri := "https://huggingface.co/mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF"
 			preferences := json.RawMessage(`{}`)
@@ -242,7 +252,7 @@ var _ = Describe("DiscoverModelConfig", func() {
 			for _, imp := range registry {
 				names = append(names, imp.Name())
 			}
-			Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers"))
+			Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers", "longcat-video"))
 		})

 		It("LlamaCPPImporter exposes name/modality/autodetect", func() {
--- a/core/gallery/importers/longcat-video.go
+++ b/core/gallery/importers/longcat-video.go
@@ -0,0 +1,127 @@
+package importers
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/schema"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	longCatOwner      = "meituan-longcat"
+	longCatBaseRepo   = "LongCat-Video"
+	longCatAvatarRepo = "LongCat-Video-Avatar-1.5"
+)
+
+var _ Importer = &LongCatVideoImporter{}
+
+// LongCatVideoImporter is deliberately owner/repository-specific. LongCat
+// checkpoints also contain generic Diffusers metadata, so broad file-based
+// matching would let this backend claim unrelated video pipelines.
+type LongCatVideoImporter struct{}
+
+func (i *LongCatVideoImporter) Name() string      { return "longcat-video" }
+func (i *LongCatVideoImporter) Modality() string  { return "video" }
+func (i *LongCatVideoImporter) AutoDetects() bool { return true }
+
+func isLongCatRepo(owner, repo string) bool {
+	repo = strings.Split(strings.Trim(repo, "/"), "/")[0]
+	return strings.EqualFold(owner, longCatOwner) &&
+		(strings.EqualFold(repo, longCatBaseRepo) || strings.EqualFold(repo, longCatAvatarRepo))
+}
+
+func longCatModelID(details Details) (string, bool) {
+	if details.HuggingFace != nil && details.HuggingFace.ModelID != "" {
+		parts := strings.SplitN(details.HuggingFace.ModelID, "/", 2)
+		if len(parts) == 2 && isLongCatRepo(parts[0], parts[1]) {
+			repo := strings.Split(strings.Trim(parts[1], "/"), "/")[0]
+			return parts[0] + "/" + repo, true
+		}
+	}
+	if owner, repo, ok := HFOwnerRepoFromURI(details.URI); ok && isLongCatRepo(owner, repo) {
+		repo = strings.Split(strings.Trim(repo, "/"), "/")[0]
+		return owner + "/" + repo, true
+	}
+	return LocalModelPath(details.URI), false
+}
+
+func (i *LongCatVideoImporter) Match(details Details) bool {
+	preferences, err := details.Preferences.MarshalJSON()
+	if err != nil {
+		return false
+	}
+	preferencesMap := make(map[string]any)
+	if len(preferences) > 0 {
+		if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
+			return false
+		}
+	}
+	if backend, ok := preferencesMap["backend"].(string); ok {
+		return backend == i.Name()
+	}
+
+	_, matched := longCatModelID(details)
+	return matched
+}
+
+func (i *LongCatVideoImporter) Import(details Details) (gallery.ModelConfig, error) {
+	preferences, err := details.Preferences.MarshalJSON()
+	if err != nil {
+		return gallery.ModelConfig{}, err
+	}
+	preferencesMap := make(map[string]any)
+	if len(preferences) > 0 {
+		if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
+			return gallery.ModelConfig{}, err
+		}
+	}
+
+	model, canonical := longCatModelID(details)
+	name, _ := preferencesMap["name"].(string)
+	if name == "" {
+		if canonical {
+			name = strings.ToLower(filepath.Base(model))
+		} else {
+			name = filepath.Base(strings.TrimSuffix(model, "/"))
+		}
+	}
+
+	description, _ := preferencesMap["description"].(string)
+	if description == "" {
+		description = "Imported from " + details.URI
+	}
+
+	options := []string{"attention_backend:sdpa"}
+	inputModalities := []string{config.ModalityText, config.ModalityImage}
+	if strings.EqualFold(filepath.Base(model), longCatAvatarRepo) {
+		options = append(options, "use_distill:true")
+		inputModalities = append(inputModalities, config.ModalityAudio)
+	}
+
+	modelConfig := config.ModelConfig{
+		Name:                  name,
+		Description:           description,
+		Backend:               i.Name(),
+		KnownUsecaseStrings:   []string{config.UsecaseVideo},
+		KnownInputModalities:  inputModalities,
+		KnownOutputModalities: []string{config.ModalityVideo},
+		Options:               options,
+		PredictionOptions: schema.PredictionOptions{
+			BasicModelRequest: schema.BasicModelRequest{Model: model},
+		},
+	}
+
+	data, err := yaml.Marshal(modelConfig)
+	if err != nil {
+		return gallery.ModelConfig{}, err
+	}
+	return gallery.ModelConfig{
+		Name:        name,
+		Description: description,
+		ConfigFile:  string(data),
+	}, nil
+}
--- a/core/gallery/importers/longcat-video_test.go
+++ b/core/gallery/importers/longcat-video_test.go
@@ -0,0 +1,116 @@
+package importers_test
+
+import (
+	"encoding/json"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery/importers"
+	hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+func decodeLongCatModelConfig(data string) config.ModelConfig {
+	GinkgoHelper()
+	modelConfig := config.ModelConfig{}
+	Expect(yaml.Unmarshal([]byte(data), &modelConfig)).To(Succeed())
+	return modelConfig
+}
+
+var _ = Describe("LongCatVideoImporter", func() {
+	var importer *importers.LongCatVideoImporter
+
+	BeforeEach(func() {
+		importer = &importers.LongCatVideoImporter{}
+	})
+
+	It("exposes video importer metadata", func() {
+		Expect(importer.Name()).To(Equal("longcat-video"))
+		Expect(importer.Modality()).To(Equal("video"))
+		Expect(importer.AutoDetects()).To(BeTrue())
+	})
+
+	Describe("Match", func() {
+		It("matches both official repositories", func() {
+			for _, modelID := range []string{
+				"meituan-longcat/LongCat-Video",
+				"meituan-longcat/LongCat-Video-Avatar-1.5",
+			} {
+				details := importers.Details{
+					URI: "https://huggingface.co/" + modelID,
+					HuggingFace: &hfapi.ModelDetails{
+						ModelID: modelID,
+						Author:  "meituan-longcat",
+					},
+				}
+				Expect(importer.Match(details)).To(BeTrue(), modelID)
+			}
+		})
+
+		It("matches official hf URI forms without metadata", func() {
+			Expect(importer.Match(importers.Details{
+				URI: "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5/tree/main/",
+			})).To(BeTrue())
+		})
+
+		It("does not claim the same repository name under another owner", func() {
+			Expect(importer.Match(importers.Details{
+				URI: "https://huggingface.co/other-org/LongCat-Video",
+			})).To(BeFalse())
+		})
+
+		It("honors an explicit backend preference", func() {
+			Expect(importer.Match(importers.Details{
+				URI:         "/models/LongCat-Video",
+				Preferences: json.RawMessage(`{"backend":"longcat-video"}`),
+			})).To(BeTrue())
+			Expect(importer.Match(importers.Details{
+				URI:         "hf://meituan-longcat/LongCat-Video",
+				Preferences: json.RawMessage(`{"backend":"diffusers"}`),
+			})).To(BeFalse())
+		})
+	})
+
+	Describe("Import", func() {
+		It("emits a base-model video configuration", func() {
+			modelConfig, err := importer.Import(importers.Details{
+				URI: "https://huggingface.co/meituan-longcat/LongCat-Video",
+			})
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(modelConfig.Name).To(Equal("longcat-video"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("- video"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("attention_backend:sdpa"))
+			Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("use_distill:true"))
+
+			cfg := decodeLongCatModelConfig(modelConfig.ConfigFile)
+			Expect(cfg.KnownInputModalities).To(Equal([]string{
+				config.ModalityText,
+				config.ModalityImage,
+			}))
+			Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo}))
+		})
+
+		It("enables the distilled path for Avatar 1.5", func() {
+			modelConfig, err := importer.Import(importers.Details{
+				URI: "hf://meituan-longcat/LongCat-Video-Avatar-1.5",
+			})
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(modelConfig.Name).To(Equal("longcat-video-avatar-1.5"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video-Avatar-1.5"))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("use_distill:true"))
+
+			cfg := decodeLongCatModelConfig(modelConfig.ConfigFile)
+			Expect(cfg.KnownInputModalities).To(Equal([]string{
+				config.ModalityText,
+				config.ModalityImage,
+				config.ModalityAudio,
+			}))
+			Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo}))
+		})
+	})
+})
--- a/core/http/app_test.go
+++ b/core/http/app_test.go
@@ -983,7 +983,22 @@ chat_template_kwargs:
 					req.Header.Set("Authorization", bearerKey)
 					resp, err = http.DefaultClient.Do(req)
 					Expect(err).ToNot(HaveOccurred())
-					Expect(resp.StatusCode).To(Equal(200))
+					if resp.StatusCode == http.StatusBadRequest {
+						// The worker can finish between the status read and cancellation request.
+						resp, err = http.Get("http://127.0.0.1:9090/api/agent/jobs/" + jobID)
+						Expect(err).ToNot(HaveOccurred())
+						Expect(resp.StatusCode).To(Equal(http.StatusOK))
+						body, _ = io.ReadAll(resp.Body)
+						err = json.Unmarshal(body, &job)
+						Expect(err).ToNot(HaveOccurred())
+						Expect(job.Status).To(Or(
+							Equal(schema.JobStatusCompleted),
+							Equal(schema.JobStatusFailed),
+							Equal(schema.JobStatusCancelled),
+						))
+					} else {
+						Expect(resp.StatusCode).To(Equal(http.StatusOK))
+					}
 				}
 			})

--- a/core/http/endpoints/localai/api_instructions.go
+++ b/core/http/endpoints/localai/api_instructions.go
@@ -71,8 +71,9 @@ var instructionDefs = []instructionDef{
 	},
 	{
 		Name:        "video",
-		Description: "Video generation from text prompts",
+		Description: "Video generation from text prompts with optional image or audio conditioning",
 		Tags:        []string{"video"},
+		Intro:       "POST /video accepts start_image, end_image, and audio as public URL, base64, or data URI. Backend-specific tuning is passed as string values in params.",
 	},
 	{
 		Name:        "face-recognition",
--- a/core/http/endpoints/localai/video.go
+++ b/core/http/endpoints/localai/video.go
@@ -1,11 +1,12 @@
 package localai

 import (
-	"bufio"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -28,32 +29,105 @@ import (
 	"github.com/mudler/LocalAI/pkg/utils"
 )

-// Downloading user-supplied media URLs legitimately follows redirects (CDNs);
-// WithFollowRedirects still strips any credential header on a cross-host hop.
-var videoDownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects())
+const maxVideoInputBytes = 128 << 20

-func downloadFile(url string) (string, error) {
-	if err := utils.ValidateExternalURL(url); err != nil {
-		return "", fmt.Errorf("URL validation failed: %w", err)
+func newVideoDownloadClient() *http.Client {
+	client := httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects())
+	checkRedirect := client.CheckRedirect
+	// Media CDNs commonly redirect, so validate every hop rather than trusting
+	// only the URL supplied by the caller. Keep the shared redirect policy too;
+	// it bounds the chain and strips credentials on cross-origin hops.
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if err := utils.ValidateExternalURL(req.URL.String()); err != nil {
+			return fmt.Errorf("redirect URL validation failed: %w", err)
+		}
+		return checkRedirect(req, via)
+	}
+	return client
+}
+
+var videoDownloadClient = newVideoDownloadClient()
+
+func openVideoMedia(ctx context.Context, ref string) (io.ReadCloser, int64, error) {
+	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		if err := utils.ValidateExternalURL(ref); err != nil {
+			return nil, 0, fmt.Errorf("URL validation failed: %w", err)
+		}
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil)
+		if err != nil {
+			return nil, 0, fmt.Errorf("creating download request: %w", err)
+		}
+		resp, err := videoDownloadClient.Do(req)
+		if err != nil {
+			return nil, 0, fmt.Errorf("downloading media: %w", err)
+		}
+		if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+			_ = resp.Body.Close()
+			return nil, 0, fmt.Errorf("media URL returned HTTP %d", resp.StatusCode)
+		}
+		return resp.Body, resp.ContentLength, nil
 	}

-	// Get the data
-	resp, err := videoDownloadClient.Get(url)
+	encoded := ref
+	if strings.HasPrefix(ref, "data:") {
+		comma := strings.IndexByte(ref, ',')
+		if comma < 0 || !strings.Contains(strings.ToLower(ref[:comma]), ";base64") {
+			return nil, 0, fmt.Errorf("data URI must contain a base64 payload")
+		}
+		encoded = ref[comma+1:]
+	}
+	if encoded == "" {
+		return nil, 0, fmt.Errorf("media payload is empty")
+	}
+
+	decodedSize := int64(base64.StdEncoding.DecodedLen(len(encoded)))
+	return io.NopCloser(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))), decodedSize, nil
+}
+
+func stageVideoMedia(ctx context.Context, directory, ref string) (string, error) {
+	return stageVideoMediaWithLimit(ctx, directory, ref, maxVideoInputBytes)
+}
+
+func stageVideoMediaWithLimit(ctx context.Context, directory, ref string, maxBytes int64) (string, error) {
+	if ref == "" {
+		return "", nil
+	}
+
+	source, declaredSize, err := openVideoMedia(ctx, ref)
 	if err != nil {
 		return "", err
 	}
-	defer resp.Body.Close()
-
-	// Create the file
-	out, err := os.CreateTemp("", "video")
-	if err != nil {
-		return "", err
+	defer func() { _ = source.Close() }()
+	if declaredSize > maxBytes {
+		return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes)
 	}
-	defer out.Close()

-	// Write the body to file
-	_, err = io.Copy(out, resp.Body)
-	return out.Name(), err
+	output, err := os.CreateTemp(directory, "video-input-*")
+	if err != nil {
+		return "", fmt.Errorf("creating staged media file: %w", err)
+	}
+	outputPath := output.Name()
+	keep := false
+	defer func() {
+		_ = output.Close()
+		if !keep {
+			_ = os.Remove(outputPath)
+		}
+	}()
+
+	written, err := io.Copy(output, io.LimitReader(source, maxBytes+1))
+	if err != nil {
+		return "", fmt.Errorf("decoding media: %w", err)
+	}
+	if written > maxBytes {
+		return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes)
+	}
+	if err := output.Close(); err != nil {
+		return "", fmt.Errorf("closing staged media file: %w", err)
+	}
+	keep = true
+	return outputPath, nil
 }

 //
@@ -72,7 +146,7 @@ func downloadFile(url string) (string, error) {
 *
 */
 // VideoEndpoint
-// @Summary Creates a video given a prompt.
+// @Summary Creates a video from a prompt and optional image or audio conditioning.
 // @Tags video
 // @Param request body schema.VideoRequest true "query params"
 // @Success 200 {object} schema.OpenAIResponse "Response"
@@ -91,63 +165,39 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 			return echo.ErrBadRequest
 		}

-		// Stage a base64- or URL-provided image into a temp file so the
-		// backend can read it as a path. Used for both start_image and
-		// (optional) end_image. Returns the temp file path, or "" if the
-		// input is empty. Caller is responsible for the defer-cleanup.
-		stageImage := func(ref string) (string, error) {
-			if ref == "" {
-				return "", nil
-			}
-			var fileData []byte
-			var err error
-			if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
-				out, derr := downloadFile(ref)
-				if derr != nil {
-					return "", fmt.Errorf("failed downloading file: %w", derr)
-				}
-				defer os.RemoveAll(out)
-				fileData, err = os.ReadFile(out)
-				if err != nil {
-					return "", fmt.Errorf("failed reading file: %w", err)
-				}
-			} else {
-				fileData, err = base64.StdEncoding.DecodeString(ref)
-				if err != nil {
-					return "", err
-				}
-			}
-			outputFile, err := os.CreateTemp(appConfig.GeneratedContentDir, "b64")
+		stageInput := func(name, ref string) (string, error) {
+			path, err := stageVideoMedia(c.Request().Context(), appConfig.GeneratedContentDir, ref)
 			if err != nil {
-				return "", err
+				return "", echo.NewHTTPError(
+					http.StatusBadRequest,
+					fmt.Sprintf("invalid %s: %v", name, err),
+				)
 			}
-			writer := bufio.NewWriter(outputFile)
-			if _, err := writer.Write(fileData); err != nil {
-				outputFile.Close()
-				return "", err
-			}
-			if err := writer.Flush(); err != nil {
-				outputFile.Close()
-				return "", err
-			}
-			outputFile.Close()
-			return outputFile.Name(), nil
+			return path, nil
 		}

-		src, err := stageImage(input.StartImage)
+		src, err := stageInput("start_image", input.StartImage)
 		if err != nil {
 			return err
 		}
 		if src != "" {
-			defer os.RemoveAll(src)
+			defer func() { _ = os.Remove(src) }()
 		}

-		endSrc, err := stageImage(input.EndImage)
+		endSrc, err := stageInput("end_image", input.EndImage)
 		if err != nil {
 			return err
 		}
 		if endSrc != "" {
-			defer os.RemoveAll(endSrc)
+			defer func() { _ = os.Remove(endSrc) }()
+		}
+
+		audioSrc, err := stageInput("audio", input.Audio)
+		if err != nil {
+			return err
+		}
+		if audioSrc != "" {
+			defer func() { _ = os.Remove(audioSrc) }()
 		}

 		xlog.Debug("Parameter Config", "config", config)
@@ -174,13 +224,19 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 		tempDir := ""
 		if !b64JSON {
 			tempDir = filepath.Join(appConfig.GeneratedContentDir, "videos")
+			if err := os.MkdirAll(tempDir, 0o750); err != nil {
+				return err
+			}
 		}
 		// Create a temporary file
 		outputFile, err := os.CreateTemp(tempDir, "b64")
 		if err != nil {
 			return err
 		}
-		outputFile.Close()
+		if err := outputFile.Close(); err != nil {
+			_ = os.Remove(outputFile.Name())
+			return err
+		}

 		// TODO: use mime type to determine the extension
 		output := outputFile.Name() + ".mp4"
@@ -188,8 +244,15 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 		// Rename the temporary file
 		err = os.Rename(outputFile.Name(), output)
 		if err != nil {
+			_ = os.Remove(outputFile.Name())
 			return err
 		}
+		preserveOutput := false
+		defer func() {
+			if !preserveOutput {
+				_ = os.Remove(output)
+			}
+		}()

 		baseURL := middleware.BaseURL(c)

@@ -204,33 +267,36 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 			"negative_prompt", input.NegativePrompt)

 		fn, err := backend.VideoGeneration(
-			height,
-			width,
-			input.Prompt,
-			input.NegativePrompt,
-			src,
-			endSrc,
-			output,
-			input.NumFrames,
-			input.FPS,
-			input.Seed,
-			input.CFGScale,
-			input.Step,
+			backend.VideoGenerationOptions{
+				Height:         height,
+				Width:          width,
+				Prompt:         input.Prompt,
+				NegativePrompt: input.NegativePrompt,
+				StartImage:     src,
+				EndImage:       endSrc,
+				Audio:          audioSrc,
+				Destination:    output,
+				NumFrames:      input.NumFrames,
+				FPS:            input.FPS,
+				Seed:           input.Seed,
+				CFGScale:       input.CFGScale,
+				Step:           input.Step,
+				Params:         input.Params,
+			},
 			ml,
 			*config,
 			appConfig,
 		)
 		if err != nil {
-			return err
+			return mapBackendError(err)
 		}
 		if err := fn(); err != nil {
-			return err
+			return mapBackendError(err)
 		}

 		item := &schema.Item{}

 		if b64JSON {
-			defer os.RemoveAll(output)
 			data, err := os.ReadFile(output)
 			if err != nil {
 				return err
@@ -242,6 +308,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 			if err != nil {
 				return err
 			}
+			preserveOutput = true
 		}

 		id := uuid.New().String()
--- a/core/http/endpoints/localai/video_internal_test.go
+++ b/core/http/endpoints/localai/video_internal_test.go
@@ -0,0 +1,68 @@
+package localai
+
+import (
+	"context"
+	"encoding/base64"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("video media staging", func() {
+	It("stages raw base64 into a private temporary file", func() {
+		directory := GinkgoT().TempDir()
+		content := []byte("avatar audio")
+
+		path, err := stageVideoMedia(context.Background(), directory, base64.StdEncoding.EncodeToString(content))
+
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(os.Remove, path)
+		Expect(filepath.Dir(path)).To(Equal(directory))
+		Expect(os.ReadFile(path)).To(Equal(content))
+		info, err := os.Stat(path)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600)))
+	})
+
+	It("accepts browser data URIs with codec parameters", func() {
+		content := []byte("recorded speech")
+		ref := "data:audio/webm;codecs=opus;base64," + base64.StdEncoding.EncodeToString(content)
+
+		path, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), ref)
+
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(os.Remove, path)
+		Expect(os.ReadFile(path)).To(Equal(content))
+	})
+
+	It("rejects malformed base64 and removes the partial file", func() {
+		directory := GinkgoT().TempDir()
+
+		_, err := stageVideoMedia(context.Background(), directory, "not%%%base64")
+
+		Expect(err).To(MatchError(ContainSubstring("decoding media")))
+		entries, readErr := os.ReadDir(directory)
+		Expect(readErr).NotTo(HaveOccurred())
+		Expect(entries).To(BeEmpty())
+	})
+
+	It("enforces the configured streaming limit", func() {
+		directory := GinkgoT().TempDir()
+		encoded := base64.StdEncoding.EncodeToString([]byte("four"))
+
+		_, err := stageVideoMediaWithLimit(context.Background(), directory, encoded, 3)
+
+		Expect(err).To(MatchError(ContainSubstring("3-byte limit")))
+		entries, readErr := os.ReadDir(directory)
+		Expect(readErr).NotTo(HaveOccurred())
+		Expect(entries).To(BeEmpty())
+	})
+
+	It("rejects non-base64 data URIs", func() {
+		_, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), "data:audio/wav,plain")
+
+		Expect(err).To(MatchError(ContainSubstring("base64 payload")))
+	})
+})
--- a/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js
+++ b/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js
@@ -23,6 +23,7 @@ const MOCK_BACKENDS = [
  { name: 'kokoro', modality: 'tts', auto_detect: true, installed: true },
  { name: 'whisper', modality: 'asr', auto_detect: true, installed: true },
  { name: 'diffusers', modality: 'image', auto_detect: true, installed: false },
+  { name: 'longcat-video', modality: 'video', auto_detect: true, installed: false },
  { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true },
  { name: 'rerankers', modality: 'reranker', auto_detect: true, installed: true },
  { name: 'rfdetr', modality: 'detection', auto_detect: true, installed: true },
@@ -78,7 +79,7 @@ test.describe('Import form UX — Batch E (modality chip row)', () => {
    await page.locator('[data-testid="simple-options-toggle"]').click()
    await expect(chips(page)).toBeVisible()
    // Full set of chips renders.
-    for (const key of ['', 'text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad']) {
+    for (const key of ['', 'text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad']) {
      await expect(chip(page, key)).toBeVisible()
    }
    // "Any" is active by default.
--- a/core/http/react-ui/e2e/media-history.spec.js
+++ b/core/http/react-ui/e2e/media-history.spec.js
@@ -21,9 +21,10 @@ function mockImageGeneration(page, images) {
  })
 }

-function mockVideoGeneration(page, videos) {
+function mockVideoGeneration(page, videos, onRequest) {
  return page.route('**/video', (route) => {
    if (route.request().method() !== 'POST') return route.continue()
+    onRequest?.(route.request().postDataJSON())
    route.fulfill({
      contentType: 'application/json',
      body: JSON.stringify({
@@ -254,4 +255,22 @@ test.describe('Media History - Video Generation', () => {
    await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 10_000 })
    await expect(page.getByTestId('media-history-item')).toContainText('a running cat')
  })
+
+  test('Avatar audio is forwarded to the video API', async ({ page }) => {
+    let requestBody
+    await mockVideoGeneration(page, [{ url: '/generated-videos/avatar.mp4' }], (body) => { requestBody = body })
+
+    await page.goto('/app/video')
+    await expect(page.getByRole('button', { name: 'test-video-model' })).toBeVisible({ timeout: 10_000 })
+    await page.getByRole('button', { name: 'Reference media' }).click()
+    await page.getByLabel('Avatar audio', { exact: true }).setInputFiles({
+      name: 'speech.wav',
+      mimeType: 'audio/wav',
+      buffer: Buffer.from('avatar speech'),
+    })
+    await page.locator('.textarea').first().fill('a presenter speaking to camera')
+    await page.locator('button[type="submit"]').click()
+
+    await expect.poll(() => requestBody?.audio).toBe(Buffer.from('avatar speech').toString('base64'))
+  })
 })
--- a/core/http/react-ui/public/locales/de/importModel.json
+++ b/core/http/react-ui/public/locales/de/importModel.json
@@ -72,6 +72,7 @@
    "asr": "Spracherkennung",
    "tts": "Sprachsynthese",
    "image": "Bild / Video",
+    "video": "Videogenerierung",
    "embeddings": "Embeddings",
    "reranker": "Reranker",
    "detection": "Objekterkennung",
--- a/core/http/react-ui/public/locales/de/media.json
+++ b/core/http/react-ui/public/locales/de/media.json
@@ -52,7 +52,12 @@
      "size": "Größe",
      "advanced": "Erweiterte Einstellungen",
      "seed": "Seed",
-      "seedPlaceholder": "Zufällig"
+      "seedPlaceholder": "Zufällig",
+      "frames": "Frames",
+      "referenceMedia": "Referenzmedien",
+      "startImage": "Startbild",
+      "endImage": "Endbild",
+      "avatarAudio": "Avatar-Audio"
    },
    "actions": {
      "generate": "Video generieren",
--- a/core/http/react-ui/public/locales/en/importModel.json
+++ b/core/http/react-ui/public/locales/en/importModel.json
@@ -72,6 +72,7 @@
    "asr": "Speech recognition",
    "tts": "Text-to-speech",
    "image": "Image / Video",
+    "video": "Video generation",
    "embeddings": "Embeddings",
    "reranker": "Rerankers",
    "detection": "Object detection",
--- a/core/http/react-ui/public/locales/en/media.json
+++ b/core/http/react-ui/public/locales/en/media.json
@@ -52,7 +52,12 @@
      "size": "Size",
      "advanced": "Advanced Settings",
      "seed": "Seed",
-      "seedPlaceholder": "Random"
+      "seedPlaceholder": "Random",
+      "frames": "Frames",
+      "referenceMedia": "Reference media",
+      "startImage": "Start image",
+      "endImage": "End image",
+      "avatarAudio": "Avatar audio"
    },
    "actions": {
      "generate": "Generate",
--- a/core/http/react-ui/public/locales/es/importModel.json
+++ b/core/http/react-ui/public/locales/es/importModel.json
@@ -72,6 +72,7 @@
    "asr": "Reconocimiento de voz",
    "tts": "Texto a voz",
    "image": "Imagen / Video",
+    "video": "Generación de video",
    "embeddings": "Embeddings",
    "reranker": "Rerankers",
    "detection": "Detección de objetos",
--- a/core/http/react-ui/public/locales/es/media.json
+++ b/core/http/react-ui/public/locales/es/media.json
@@ -52,7 +52,12 @@
      "size": "Tamaño",
      "advanced": "Configuración avanzada",
      "seed": "Seed",
-      "seedPlaceholder": "Aleatorio"
+      "seedPlaceholder": "Aleatorio",
+      "frames": "Fotogramas",
+      "referenceMedia": "Medios de referencia",
+      "startImage": "Imagen inicial",
+      "endImage": "Imagen final",
+      "avatarAudio": "Audio del avatar"
    },
    "actions": {
      "generate": "Generar video",
--- a/core/http/react-ui/public/locales/id/importModel.json
+++ b/core/http/react-ui/public/locales/id/importModel.json
@@ -72,6 +72,7 @@
    "asr": "Pengenalan suara",
    "tts": "Text-to-speech",
    "image": "Gambar / Video",
+    "video": "Pembuatan video",
    "embeddings": "Embedding",
    "reranker": "Reranker",
    "detection": "Deteksi object",
--- a/core/http/react-ui/public/locales/id/media.json
+++ b/core/http/react-ui/public/locales/id/media.json
@@ -52,7 +52,12 @@
      "size": "Ukuran",
      "advanced": "Pengaturan Tingkat Lanjutan",
      "seed": "Seed",
-      "seedPlaceholder": "Acak"
+      "seedPlaceholder": "Acak",
+      "frames": "Frame",
+      "referenceMedia": "Media referensi",
+      "startImage": "Gambar awal",
+      "endImage": "Gambar akhir",
+      "avatarAudio": "Audio avatar"
    },
    "actions": {
      "generate": "Hasilkan",
--- a/core/http/react-ui/public/locales/it/importModel.json
+++ b/core/http/react-ui/public/locales/it/importModel.json
@@ -72,6 +72,7 @@
    "asr": "Riconoscimento vocale",
    "tts": "Sintesi vocale",
    "image": "Immagine / Video",
+    "video": "Generazione video",
    "embeddings": "Embedding",
    "reranker": "Reranker",
    "detection": "Rilevamento oggetti",
--- a/core/http/react-ui/public/locales/it/media.json
+++ b/core/http/react-ui/public/locales/it/media.json
@@ -52,7 +52,12 @@
      "size": "Dimensione",
      "advanced": "Impostazioni avanzate",
      "seed": "Seed",
-      "seedPlaceholder": "Casuale"
+      "seedPlaceholder": "Casuale",
+      "frames": "Fotogrammi",
+      "referenceMedia": "Media di riferimento",
+      "startImage": "Immagine iniziale",
+      "endImage": "Immagine finale",
+      "avatarAudio": "Audio avatar"
    },
    "actions": {
      "generate": "Genera video",
--- a/core/http/react-ui/public/locales/ko/importModel.json
+++ b/core/http/react-ui/public/locales/ko/importModel.json
@@ -72,6 +72,7 @@
    "asr": "음성 인식",
    "tts": "텍스트 음성 변환",
    "image": "이미지 / 비디오",
+    "video": "비디오 생성",
    "embeddings": "임베딩",
    "reranker": "리랭커",
    "detection": "객체 감지",
--- a/core/http/react-ui/public/locales/ko/media.json
+++ b/core/http/react-ui/public/locales/ko/media.json
@@ -52,7 +52,12 @@
      "size": "크기",
      "advanced": "고급 설정",
      "seed": "시드",
-      "seedPlaceholder": "랜덤"
+      "seedPlaceholder": "랜덤",
+      "frames": "프레임",
+      "referenceMedia": "참조 미디어",
+      "startImage": "시작 이미지",
+      "endImage": "종료 이미지",
+      "avatarAudio": "아바타 오디오"
    },
    "actions": {
      "generate": "생성",
--- a/core/http/react-ui/public/locales/zh-CN/importModel.json
+++ b/core/http/react-ui/public/locales/zh-CN/importModel.json
@@ -72,6 +72,7 @@
    "asr": "语音识别",
    "tts": "文字转语音",
    "image": "图像 / 视频",
+    "video": "视频生成",
    "embeddings": "嵌入",
    "reranker": "重排器",
    "detection": "对象检测",
--- a/core/http/react-ui/public/locales/zh-CN/media.json
+++ b/core/http/react-ui/public/locales/zh-CN/media.json
@@ -52,7 +52,12 @@
      "size": "尺寸",
      "advanced": "高级设置",
      "seed": "随机种子",
-      "seedPlaceholder": "随机"
+      "seedPlaceholder": "随机",
+      "frames": "帧数",
+      "referenceMedia": "参考媒体",
+      "startImage": "起始图像",
+      "endImage": "结束图像",
+      "avatarAudio": "虚拟形象音频"
    },
    "actions": {
      "generate": "生成视频",
--- a/core/http/react-ui/src/App.css
+++ b/core/http/react-ui/src/App.css
@@ -2605,6 +2605,24 @@ select.input {
  user-select: none;
 }

+button.collapsible-header {
+  width: 100%;
+  border: 0;
+  background: transparent;
+  font-family: inherit;
+  text-align: left;
+}
+
+button.collapsible-header:hover {
+  color: var(--color-text-primary);
+}
+
+button.collapsible-header:focus-visible {
+  border-radius: var(--radius-sm);
+  outline: 2px solid var(--color-primary);
+  outline-offset: 2px;
+}
+
 .collapsible-header i {
  transition: transform var(--duration-fast);
 }
--- a/core/http/react-ui/src/components/ModalityChips.jsx
+++ b/core/http/react-ui/src/components/ModalityChips.jsx
@@ -16,6 +16,7 @@ const CHIPS = [
  { key: 'asr', label: 'Speech' },
  { key: 'tts', label: 'TTS' },
  { key: 'image', label: 'Image' },
+  { key: 'video', label: 'Video' },
  { key: 'embeddings', label: 'Embeddings' },
  { key: 'reranker', label: 'Rerankers' },
  { key: 'detection', label: 'Detection' },
--- a/core/http/react-ui/src/pages/ImportModel.jsx
+++ b/core/http/react-ui/src/pages/ImportModel.jsx
@@ -16,7 +16,7 @@ const BACKENDS_FALLBACK_EMPTY = []

 // Modality keys used as i18n keys under "modality.*" namespace; resolved
 // at render time inside `buildBackendOptions`.
-const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad']
+const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad']

 // buildBackendOptions groups known backends by modality and tags
 // auto_detect=false entries with a muted "manual pick" badge so users
--- a/core/http/react-ui/src/pages/VideoGen.jsx
+++ b/core/http/react-ui/src/pages/VideoGen.jsx
@@ -8,10 +8,11 @@ import LoadingSpinner from '../components/LoadingSpinner'
 import GenerationProgress from '../components/GenerationProgress'
 import ErrorWithTraceLink from '../components/ErrorWithTraceLink'
 import MediaHistory from '../components/MediaHistory'
+import MediaInput from '../components/biometrics/MediaInput'
 import { videoApi, fileToBase64 } from '../utils/api'
 import { useMediaHistory } from '../hooks/useMediaHistory'

-const SIZES = ['256x256', '512x512', '768x768', '1024x1024']
+const SIZES = ['256x256', '512x512', '768x768', '1024x1024', '832x480', '1280x720']

 export default function VideoGen() {
  const { model: urlModel } = useParams()
@@ -31,9 +32,10 @@ export default function VideoGen() {
  const [error, setError] = useState(null)
  const [videos, setVideos] = useState([])
  const [showAdvanced, setShowAdvanced] = useState(false)
-  const [showImageInputs, setShowImageInputs] = useState(false)
+  const [showMediaInputs, setShowMediaInputs] = useState(false)
  const [startImage, setStartImage] = useState(null)
  const [endImage, setEndImage] = useState(null)
+  const [audioInput, setAudioInput] = useState(null)
  const { addEntry, selectEntry, selectedEntry, historyProps } = useMediaHistory('video')

  const handleGenerate = async (e) => {
@@ -55,6 +57,7 @@ export default function VideoGen() {
    if (cfgScale) body.cfg_scale = parseFloat(cfgScale)
    if (startImage) body.start_image = startImage
    if (endImage) body.end_image = endImage
+    if (audioInput?.base64) body.audio = audioInput.base64

    try {
      const data = await videoApi.generate(body)
@@ -116,24 +119,46 @@ export default function VideoGen() {
            </div>
          </div>

-          <div className={`collapsible-header ${showAdvanced ? 'open' : ''}`} onClick={() => setShowAdvanced(!showAdvanced)}>
-            <i className="fas fa-chevron-right" /> {t('video.labels.advanced')}
-          </div>
+          <button
+            type="button"
+            className={`collapsible-header ${showAdvanced ? 'open' : ''}`}
+            aria-expanded={showAdvanced}
+            aria-controls="video-advanced-options"
+            onClick={() => setShowAdvanced(!showAdvanced)}
+          >
+            <i className="fas fa-chevron-right" aria-hidden="true" /> {t('video.labels.advanced')}
+          </button>
          {showAdvanced && (
-            <div className="form-grid-3col">
+            <div id="video-advanced-options" className="form-grid-3col">
              <div className="form-group"><label className="form-label">{t('image.labels.steps')}</label><input className="input" type="number" value={steps} onChange={(e) => setSteps(e.target.value)} /></div>
              <div className="form-group"><label className="form-label">{t('video.labels.seed')}</label><input className="input" type="number" value={seed} onChange={(e) => setSeed(e.target.value)} /></div>
              <div className="form-group"><label className="form-label">CFG Scale</label><input className="input" type="number" step="0.1" value={cfgScale} onChange={(e) => setCfgScale(e.target.value)} /></div>
+              <div className="form-group"><label className="form-label">{t('video.labels.frames')}</label><input className="input" type="number" min="1" value={frames} onChange={(e) => setFrames(e.target.value)} /></div>
            </div>
          )}

-          <div className={`collapsible-header ${showImageInputs ? 'open' : ''}`} onClick={() => setShowImageInputs(!showImageInputs)}>
-            <i className="fas fa-chevron-right" /> {t('image.labels.imageInputs')}
-          </div>
-          {showImageInputs && (
-            <div className="form-grid-2col">
-              <div className="form-group"><label className="form-label">Start Image</label><input type="file" accept="image/*" onChange={(e) => handleImageUpload(e, setStartImage)} className="input" /></div>
-              <div className="form-group"><label className="form-label">End Image</label><input type="file" accept="image/*" onChange={(e) => handleImageUpload(e, setEndImage)} className="input" /></div>
+          <button
+            type="button"
+            className={`collapsible-header ${showMediaInputs ? 'open' : ''}`}
+            aria-expanded={showMediaInputs}
+            aria-controls="video-reference-media"
+            onClick={() => setShowMediaInputs(!showMediaInputs)}
+          >
+            <i className="fas fa-chevron-right" aria-hidden="true" /> {t('video.labels.referenceMedia')}
+          </button>
+          {showMediaInputs && (
+            <div id="video-reference-media">
+              <div className="form-grid-2col">
+                <div className="form-group"><label className="form-label">{t('video.labels.startImage')}</label><input type="file" accept="image/*" onChange={(e) => handleImageUpload(e, setStartImage)} className="input" /></div>
+                <div className="form-group"><label className="form-label">{t('video.labels.endImage')}</label><input type="file" accept="image/*" onChange={(e) => handleImageUpload(e, setEndImage)} className="input" /></div>
+              </div>
+              <MediaInput
+                mode="audio"
+                label={t('video.labels.avatarAudio')}
+                value={audioInput}
+                onChange={setAudioInput}
+                idPrefix="video-avatar"
+              />
            </div>
          )}

--- a/core/schema/localai.go
+++ b/core/schema/localai.go
@@ -55,6 +55,7 @@ type VideoRequest struct {
 	NegativePrompt string  `json:"negative_prompt" yaml:"negative_prompt"`                   // things to avoid in the output
 	StartImage     string  `json:"start_image" yaml:"start_image"`                           // URL or base64 of the first frame
 	EndImage       string  `json:"end_image" yaml:"end_image"`                               // URL or base64 of the last frame
+	Audio          string  `json:"audio,omitempty" yaml:"audio,omitempty"`                   // URL or base64 audio for audio-conditioned generation
 	Width          int32   `json:"width" yaml:"width"`                                       // output width in pixels
 	Height         int32   `json:"height" yaml:"height"`                                     // output height in pixels
 	NumFrames      int32   `json:"num_frames" yaml:"num_frames"`                             // total number of frames to generate
@@ -66,6 +67,7 @@ type VideoRequest struct {
 	CFGScale       float32 `json:"cfg_scale" yaml:"cfg_scale"`                               // classifier-free guidance scale
 	Step           int32   `json:"step" yaml:"step"`                                         // number of diffusion steps
 	ResponseFormat string  `json:"response_format" yaml:"response_format"`                   // output format (url or b64_json)
+	Params         map[string]string `json:"params,omitempty" yaml:"params,omitempty"`         // backend-specific generation parameters
 }

 // @Description TTS request body
--- a/core/services/nodes/file_staging_client.go
+++ b/core/services/nodes/file_staging_client.go
@@ -162,7 +162,7 @@ func (f *FileStagingClient) GenerateImage(ctx context.Context, in *pb.GenerateIm
 func (f *FileStagingClient) GenerateVideo(ctx context.Context, in *pb.GenerateVideoRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
 	reqID := requestID()

-	// Stage start/end images
+	// Stage start/end images and optional audio conditioning.
 	if in.StartImage != "" && isFilePath(in.StartImage) {
 		backendPath, _, err := f.stageInputFile(ctx, reqID, in.StartImage, "inputs")
 		if err != nil {
@@ -177,6 +177,13 @@ func (f *FileStagingClient) GenerateVideo(ctx context.Context, in *pb.GenerateVi
 		}
 		in.EndImage = backendPath
 	}
+	if in.Audio != "" && isFilePath(in.Audio) {
+		backendPath, _, err := f.stageInputFile(ctx, reqID, in.Audio, "inputs")
+		if err != nil {
+			return nil, fmt.Errorf("staging video audio: %w", err)
+		}
+		in.Audio = backendPath
+	}

 	// Handle output destination
 	frontendDst := in.Dst
--- a/docs/content/advanced/model-configuration.md
+++ b/docs/content/advanced/model-configuration.md
@@ -875,6 +875,23 @@ Available flags: `chat`, `completion`, `edit`, `embeddings`, `rerank`, `image`,

 `token_classify` marks a model as a token-classification (NER) provider for the PII filter (e.g. an `openai-privacy-filter` GGUF). Declare it explicitly together with `embeddings: true` (the classifier loads via TOKEN_CLS pooling). It runs on the dedicated `privacy-filter` backend (`backend/cpp/privacy-filter`), a standalone GGML engine for the `openai-privacy-filter` family — separate from `llama-cpp`, which no longer carries the token-classification path.

+### Known input and output modalities
+
+Use `known_input_modalities` and `known_output_modalities` when a use case does not fully describe a model's I/O. For example, both text-to-video and audio-driven avatar models use the `video` use case, but only the avatar model accepts audio:
+
+```yaml
+known_usecases:
+  - video
+known_input_modalities:
+  - text
+  - image
+  - audio
+known_output_modalities:
+  - video
+```
+
+Valid modality values are `text`, `image`, `audio`, and `video`. Explicit values are combined with modalities LocalAI can infer from the model use cases and configuration. The resulting canonical, de-duplicated lists are exposed by `GET /v1/models/capabilities`.
+
 ## PII filtering

 PII redaction is NER-based and runs on the **request** (input) side. It has two halves:
--- a/docs/content/features/_index.en.md
+++ b/docs/content/features/_index.en.md
@@ -17,7 +17,8 @@ LocalAI provides a comprehensive set of features for running AI models locally.
 - **[Text to Audio](text-to-audio/)** - Generate speech from text with TTS models
 - **[Sound Generation](sound-generation/)** - Generate music and sound effects from text descriptions
 - **[Voice Activity Detection](voice-activity-detection/)** - Detect speech segments in audio data
- **[Video Generation](video-generation/)** - Generate videos from text prompts and reference images
+- **[Video Generation](video-generation/)** - Generate videos from text prompts with optional image or audio conditioning
+- **[LongCat Video and Avatar](longcat-video/)** - Run text/image-to-video and audio-driven avatars on NVIDIA CUDA and DGX Spark
 - **[Embeddings](embeddings/)** - Generate vector embeddings for semantic search and RAG applications
 - **[GPT Vision](gpt-vision/)** - Analyze and understand images with vision-language models

--- a/docs/content/features/api-discovery.md
+++ b/docs/content/features/api-discovery.md
@@ -159,7 +159,7 @@ curl http://localhost:8080/v1/models/capabilities
 ```

 - **`capabilities`** — canonical usecase strings (e.g. `chat`, `vision`, `transcript`, `tts`, `embeddings`, `image`, `video`) plus the modifiers `tools` and `thinking`.
- **`input_modalities` / `output_modalities`** — the subsets of `{text, image, audio, video}` the model accepts and produces. Audio and video *input* are derived from the model's multimodal limits (e.g. vLLM `limit_mm_per_prompt`), which no single usecase flag expresses — which is why this endpoint exists alongside the plain listing.
+- **`input_modalities` / `output_modalities`** — the subsets of `{text, image, audio, video}` the model accepts and produces. LocalAI combines usecase-based inference, backend settings such as vLLM `limit_mm_per_prompt`, and explicit model-level `known_input_modalities` / `known_output_modalities`. The explicit fields cover distinctions a usecase cannot express, such as a video model that also accepts speech.

 The same query parameters as `/v1/models` are honored (`filter`, `excludeConfigured`), and the same per-user model allowlist is applied when authentication is enabled.

--- a/docs/content/features/backends.md
+++ b/docs/content/features/backends.md
@@ -129,9 +129,9 @@ LocalAI supports various types of backends:
 - **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS)
 - **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step)
 - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp)
- **Image & Video Generation Backends**: For diffusion models (e.g., stable-diffusion.cpp, diffusers)
+- **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [LongCat-Video]({{%relref "features/longcat-video" %}}))
 - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface)
 - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE)
 - **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store)

-See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog.
+See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog.
--- a/docs/content/features/longcat-video.md
+++ b/docs/content/features/longcat-video.md
@@ -0,0 +1,169 @@
+++
+disableToc = false
+title = "LongCat Video and Avatar"
+weight = 19
+url = "/features/longcat-video/"
+++
+
+LocalAI's `longcat-video` backend serves Meituan's official LongCat video-generation models through the `/video` API and the Studio **Video** page.
+
+| Gallery model | Upstream checkpoint | Inputs | Output |
+|---------------|---------------------|--------|--------|
+| `longcat-video` | `meituan-longcat/LongCat-Video` | text, optional start image | video |
+| `longcat-video-avatar-1.5` | `meituan-longcat/LongCat-Video-Avatar-1.5` | text, audio, optional portrait | video with the source audio |
+
+The base checkpoint supports text-to-video and image-to-video. Avatar 1.5 adds audio-driven character animation, optional portrait conditioning, and continuation segments for longer speech.
+
+{{% notice warning %}}
+LongCat is a large, CUDA-only model family. LocalAI publishes this backend for Linux with NVIDIA CUDA 12 or CUDA 13 on x86_64 and CUDA 13 on ARM64. CPU, ROCm, and macOS images are not available. Avatar 1.5 also loads components from the base checkpoint, so reserve substantial disk and GPU or unified memory.
+{{% /notice %}}
+
+## Install from the Model Gallery
+
+Install one or both recipes from **Models** in the web UI, or use the CLI:
+
+```bash
+local-ai models install longcat-video
+local-ai models install longcat-video-avatar-1.5
+```
+
+You can also import either official Hugging Face URL. The importer recognizes the two repositories and writes a `longcat-video` model config with the appropriate use case and input/output modalities.
+
+The required OCI backend is installed automatically when LocalAI first loads the model. The hardware detector selects the CUDA 12, CUDA 13, or CUDA 13 ARM64 variant.
+
+### DGX Spark and NVIDIA ARM64
+
+Use a LocalAI CUDA 13 ARM64 image as described in [GPU acceleration]({{%relref "features/GPU-acceleration" %}}). The backend defaults to PyTorch SDPA, avoiding the FlashAttention dependency that is commonly unavailable on Blackwell ARM64 systems.
+
+For unified-memory systems, start with BF16 (`use_int8:false`, the default). INT8 lowers steady-state DiT memory but can have a higher load-time peak because the full model is materialized before the quantized weights are applied.
+
+## Generate in Studio
+
+1. Open **Studio**, then choose **Video**.
+2. Select `longcat-video` or `longcat-video-avatar-1.5`.
+3. Enter a prompt and choose `832x480` or `1280x720`.
+4. Expand **Reference media** to upload a start image. For Avatar 1.5, upload or record the speech under **Avatar audio**.
+5. Select **Generate**.
+
+The base model can run without a reference image for text-to-video. Avatar 1.5 requires audio; the portrait is optional.
+
+## API examples
+
+### Text-to-video
+
+```bash
+curl http://localhost:8080/video \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "longcat-video",
+    "prompt": "A cinematic tracking shot through a misty redwood forest",
+    "width": 832,
+    "height": 480,
+    "num_frames": 93,
+    "fps": 15
+  }'
+```
+
+### Image-to-video
+
+`start_image` accepts raw base64, a browser-style data URI, or a public HTTP(S) URL:
+
+```bash
+curl http://localhost:8080/video \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"model\": \"longcat-video\",
+    \"prompt\": \"The subject turns toward the camera as leaves move in the breeze\",
+    \"start_image\": \"$(base64 --wrap=0 portrait.png)\",
+    \"params\": {
+      \"resolution\": \"480p\"
+    }
+  }"
+```
+
+### Avatar from speech and a portrait
+
+`audio` accepts raw base64, a data URI, or a public HTTP(S) URL. Each staged image or audio input is limited to 128 MiB.
+
+```bash
+curl http://localhost:8080/video \
+  -H "Content-Type: application/json" \
+  -d "{
+    \"model\": \"longcat-video-avatar-1.5\",
+    \"prompt\": \"A friendly presenter speaking naturally to camera\",
+    \"start_image\": \"$(base64 --wrap=0 portrait.png)\",
+    \"audio\": \"$(base64 --wrap=0 speech.wav)\",
+    \"width\": 832,
+    \"height\": 480,
+    \"params\": {
+      \"offload_kv_cache\": \"true\"
+    }
+  }"
+```
+
+Avatar output is generated at 25 FPS and is muxed with the submitted audio. When neither `num_frames` nor `params.num_segments` is provided, LocalAI derives the continuation count from the audio duration, up to the model's `max_segments` setting.
+
+## Model configuration
+
+The gallery and importer make each model self-describing. A manual Avatar 1.5 config looks like this:
+
+```yaml
+name: longcat-video-avatar-1.5
+backend: longcat-video
+known_usecases:
+  - video
+known_input_modalities:
+  - text
+  - image
+  - audio
+known_output_modalities:
+  - video
+options:
+  - attention_backend:sdpa
+  - use_distill:true
+  - max_segments:8
+parameters:
+  model: meituan-longcat/LongCat-Video-Avatar-1.5
+```
+
+The explicit modality declarations are used by `GET /v1/models/capabilities` and attachment-aware clients. They avoid inferring model behavior from backend or checkpoint names.
+
+### Load options
+
+Model load options use `key:value` entries in `options`:
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`; packaged images guarantee `sdpa` |
+| `use_distill` | Avatar: `true`; base: `false` | Use the checkpoint's accelerated distillation path |
+| `use_int8` | `false` | Use Avatar 1.5's INT8 DiT; unsupported by the base model |
+| `base_model` | `meituan-longcat/LongCat-Video` | Base tokenizer, text encoder, and VAE used by Avatar 1.5 |
+| `max_segments` | `8` | Maximum continuation segments accepted for one request |
+| `resolution` | `480p` | Default image-conditioned resolution: `480p` or `720p` |
+
+The initial backend supports one GPU per process. Tensor or context parallel sizes above one are rejected.
+
+### Per-request parameters
+
+The `/video` request's `params` object accepts string values:
+
+| Parameter | Description |
+|-----------|-------------|
+| `num_segments` | Explicit number of Avatar continuation segments |
+| `audio_guidance_scale` | Audio classifier-free guidance when distillation is disabled |
+| `offload_kv_cache` | Offload continuation KV cache (`true` or `false`) |
+| `ref_img_index` | Reference-frame index used during continuation |
+| `mask_frame_range` | Number of frames blended around continuation boundaries |
+| `resolution` | Per-request image-conditioned resolution (`480p` or `720p`) |
+
+With distillation enabled, Avatar uses eight inference steps and fixed text/audio guidance of `1.0`. Disable `use_distill` in the model config before tuning `step`, `cfg_scale`, or `audio_guidance_scale`.
+
+## Troubleshooting
+
+- **HTTP 400, audio is required**: Avatar 1.5 was selected without `audio`.
+- **HTTP 400, request needs too many segments**: trim the audio or raise `max_segments` in the model options.
+- **HTTP 412**: the installed LocalAI runtime cannot select a compatible NVIDIA backend image.
+- **Out of memory while loading**: use BF16 on unified-memory hardware, close other GPU workloads, or reduce model concurrency. INT8 is not guaranteed to reduce peak load memory.
+- **Slow first request**: the backend and checkpoints are downloaded and loaded on demand; subsequent requests reuse the loaded pipeline.
+
+See the general [`/video` API reference]({{%relref "features/video-generation" %}}) for the complete request and response schema.
--- a/docs/content/features/video-generation.md
+++ b/docs/content/features/video-generation.md
@@ -5,7 +5,7 @@ weight = 18
 url = "/features/video-generation/"
 +++

-LocalAI can generate videos from text prompts and optional reference images via the `/video` endpoint. Supported backends include `diffusers`, `stablediffusion`, and `vllm-omni`.
+LocalAI can generate videos from text prompts and optional image or audio conditioning via the `/video` endpoint. Supported backends include `diffusers`, `stablediffusion`, `vllm-omni`, and the dedicated `longcat-video` backend.

 ## API

@@ -23,6 +23,7 @@ The request body is JSON with the following fields:
 | `negative_prompt` | `string` | No       |         | What to exclude from the generated video                 |
 | `start_image`     | `string` | No       |         | Starting image as base64 string or URL                   |
 | `end_image`       | `string` | No       |         | Ending image for guided generation                       |
+| `audio`           | `string` | No       |         | Audio conditioning as base64, a data URI, or URL          |
 | `width`           | `int`    | No       | 512     | Video width in pixels                                    |
 | `height`          | `int`    | No       | 512     | Video height in pixels                                   |
 | `num_frames`      | `int`    | No       |         | Number of frames                                         |
@@ -34,6 +35,7 @@ The request body is JSON with the following fields:
 | `cfg_scale`       | `float`  | No       |         | Classifier-free guidance scale                           |
 | `step`            | `int`    | No       |         | Number of inference steps                                |
 | `response_format` | `string` | No       | `url`   | `url` to return a file URL, `b64_json` for base64 output |
+| `params`          | `object` | No       |         | Backend-specific string parameters                        |

 ### Response

@@ -107,9 +109,14 @@ curl http://localhost:8080/video \
  }'
 ```

+## LongCat-Video and Avatar 1.5
+
+The dedicated `longcat-video` backend serves the official base and Avatar 1.5 checkpoints, including CUDA 13 ARM64 systems such as DGX Spark. See [LongCat Video and Avatar]({{%relref "features/longcat-video" %}}) for gallery installation, Studio instructions, complete model YAML, API examples, tuning options, and hardware requirements.
+
 ## Error Responses

 | Status Code | Description                                          |
 |-------------|------------------------------------------------------|
 | 400         | Missing or invalid model or request parameters       |
+| 412         | The selected backend cannot run on the available hardware |
 | 500         | Backend error during video generation                |
--- a/docs/content/whats-new.md
+++ b/docs/content/whats-new.md
@@ -12,12 +12,13 @@ You can see the release notes [here](https://github.com/mudler/LocalAI/releases)

 ## 2026 Highlights

+- **July 2026**: [LongCat video and avatar generation](/features/longcat-video/) — dedicated CUDA backend for `LongCat-Video` text/image-to-video and `LongCat-Video-Avatar-1.5` speech-driven avatars. Includes multi-segment continuation, portrait and recorded-audio inputs in Studio, and an SDPA CUDA 13 ARM64 build for DGX Spark.
 - **April 2026**: [Audio Transform](/features/audio-transform/) — generic audio-in / audio-out endpoint with optional reference signal. First implementation: [LocalVQE](https://github.com/localai-org/LocalVQE) C++ backend (joint AEC + noise suppression + dereverberation, DeepVQE-style). Both batch (`POST /audio/transformations`) and bidirectional WebSocket streaming (`/audio/transformations/stream`). Studio "Transform" tab with synchronized waveform players for input / reference / output.
 - **April 2026**: [Face recognition backend](/features/face-recognition/) — `insightface`-powered 1:1 verification, 1:N identification, face embedding, face detection, and demographic analysis. Ships both a non-commercial `buffalo_l` model and an Apache 2.0 OpenCV Zoo alternative.
 - **May 2026**: [Speaker diarization](/features/audio-diarization/) — new `/v1/audio/diarization` endpoint returning "who spoke when" segments. Backed by `sherpa-onnx` (pyannote-3.0 + speaker embeddings + clustering) for pure diarization, and `vibevoice-cpp` for diarization bundled with long-form ASR. Supports `json` / `verbose_json` / `rttm` response formats.
 - **June 2026**: [Sound classification](/features/audio-classification/) — new `/v1/audio/classification` endpoint for audio tagging / sound-event classification, returning scored [AudioSet](https://research.google.com/audioset/) labels (baby cry, glass breaking, alarms, ...). Backed by [ced.cpp](https://github.com/mudler/ced.cpp), a 527-class AudioSet tagger ported to ggml.
 - **June 2026**: [PII analyze / redact API](/features/middleware/#analyze--redact-api) — the PII detection pipeline (NER + restricted-regex pattern tiers) is now a standalone service: `POST /api/pii/analyze` returns detected entity spans and `POST /api/pii/redact` returns the sanitised text (or `400 pii_blocked`), without routing a chat request through the middleware. Events gain an `origin` (`middleware` / `proxy` / `pii_analyze` / `pii_redact`) so `/api/pii/events` can be filtered by source.
- **July 2026**: [Model capabilities endpoint](/features/api-discovery/#model-capabilities) — `GET /v1/models/capabilities`, an additive superset of `/v1/models` that reports each model's `capabilities` plus its `input_modalities` / `output_modalities` (`text` / `image` / `audio` / `video`). Lets clients route image/audio/video attachments to a model only when it can handle them; audio/video *input* is derived from the model's multimodal limits, which no single usecase flag expresses.
+- **July 2026**: [Model capabilities endpoint](/features/api-discovery/#model-capabilities) — `GET /v1/models/capabilities`, an additive superset of `/v1/models` that reports each model's `capabilities` plus its `input_modalities` / `output_modalities` (`text` / `image` / `audio` / `video`). Lets clients route attachments using inferred or explicitly declared model modalities instead of backend-name checks.
 - **June 2026**: Concurrent scoring and PII NER on llama.cpp — the `Score` (router classifier) and `TokenClassify` (PII NER) primitives now ride llama.cpp's server task queue instead of locking the context, so they run concurrently with chat/completion/embedding traffic and with each other. The `known_usecases` restriction that forced dedicated scorer/NER model configs on llama-cpp is lifted, repeated scoring calls reuse the prompt KV cache across candidates, and scoring inputs are no longer capped by the physical batch size.

 ## 2024 Highlights
--- a/gallery/index.yaml
+++ b/gallery/index.yaml
@@ -5164,6 +5164,82 @@
      - video
    parameters:
      model: Wan-AI/Wan2.2-I2V-A14B-Diffusers
+- name: longcat-video
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/meituan-longcat/LongCat-Video
+    - https://github.com/meituan-longcat/LongCat-Video
+  description: |
+    LongCat-Video served by LocalAI's dedicated CUDA backend. Generates video
+    from a text prompt or a start image. The SDPA attention path works without
+    FlashAttention and is suitable for CUDA 13 ARM64 systems such as DGX Spark.
+
+    This is a very large checkpoint (roughly 83 GB in Hugging Face storage) and
+    requires Linux with an NVIDIA CUDA GPU plus substantial memory and disk.
+  license: mit
+  icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg
+  tags:
+    - text-to-video
+    - image-to-video
+    - video-generation
+    - longcat-video
+    - cuda
+    - gpu
+    - dgx-spark
+  last_checked: "2026-07-12"
+  overrides:
+    backend: longcat-video
+    known_usecases:
+      - video
+    known_input_modalities:
+      - text
+      - image
+    known_output_modalities:
+      - video
+    options:
+      - attention_backend:sdpa
+    parameters:
+      model: meituan-longcat/LongCat-Video
+- name: longcat-video-avatar-1.5
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5
+    - https://github.com/meituan-longcat/LongCat-Video
+  description: |
+    LongCat-Video-Avatar-1.5 served by LocalAI's dedicated CUDA backend. Turns
+    speech plus a prompt into an avatar video, optionally conditioning on a
+    portrait, and continues across multiple segments for longer audio.
+
+    Avatar generation also loads tokenizer, text encoder, and VAE components
+    from LongCat-Video. Plan for very large downloads and substantial NVIDIA
+    GPU or unified memory; CPU and macOS execution are unsupported.
+  license: mit
+  icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg
+  tags:
+    - audio-to-video
+    - image-to-video
+    - avatar-generation
+    - talking-head
+    - longcat-video
+    - cuda
+    - gpu
+    - dgx-spark
+  last_checked: "2026-07-12"
+  overrides:
+    backend: longcat-video
+    known_usecases:
+      - video
+    known_input_modalities:
+      - text
+      - image
+      - audio
+    known_output_modalities:
+      - video
+    options:
+      - attention_backend:sdpa
+      - use_distill:true
+    parameters:
+      model: meituan-longcat/LongCat-Video-Avatar-1.5
 - name: vllm-omni-qwen3-omni-30b
  url: github:mudler/LocalAI/gallery/virtual.yaml@master
  urls:
--- a/swagger/docs.go
+++ b/swagger/docs.go
@@ -3194,7 +3194,7 @@ const docTemplate = `{
                "tags": [
                    "video"
                ],
-                "summary": "Creates a video given a prompt.",
+                "summary": "Creates a video from a prompt and optional image or audio conditioning.",
                "parameters": [
                    {
                        "description": "query params",
@@ -6589,6 +6589,10 @@ const docTemplate = `{
        "schema.VideoRequest": {
            "type": "object",
            "properties": {
+                "audio": {
+                    "description": "URL or base64 audio for audio-conditioned generation",
+                    "type": "string"
+                },
                "cfg_scale": {
                    "description": "classifier-free guidance scale",
                    "type": "number"
@@ -6620,6 +6624,13 @@ const docTemplate = `{
                    "description": "total number of frames to generate",
                    "type": "integer"
                },
+                "params": {
+                    "description": "backend-specific generation parameters",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
                "prompt": {
                    "description": "text description of the video to generate",
                    "type": "string"
--- a/swagger/swagger.json
+++ b/swagger/swagger.json
@@ -3191,7 +3191,7 @@
                "tags": [
                    "video"
                ],
-                "summary": "Creates a video given a prompt.",
+                "summary": "Creates a video from a prompt and optional image or audio conditioning.",
                "parameters": [
                    {
                        "description": "query params",
@@ -6586,6 +6586,10 @@
        "schema.VideoRequest": {
            "type": "object",
            "properties": {
+                "audio": {
+                    "description": "URL or base64 audio for audio-conditioned generation",
+                    "type": "string"
+                },
                "cfg_scale": {
                    "description": "classifier-free guidance scale",
                    "type": "number"
@@ -6617,6 +6621,13 @@
                    "description": "total number of frames to generate",
                    "type": "integer"
                },
+                "params": {
+                    "description": "backend-specific generation parameters",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
                "prompt": {
                    "description": "text description of the video to generate",
                    "type": "string"
--- a/swagger/swagger.yaml
+++ b/swagger/swagger.yaml
@@ -2371,6 +2371,9 @@ definitions:
    type: object
  schema.VideoRequest:
    properties:
+      audio:
+        description: URL or base64 audio for audio-conditioned generation
+        type: string
      cfg_scale:
        description: classifier-free guidance scale
        type: number
@@ -2394,6 +2397,11 @@ definitions:
      num_frames:
        description: total number of frames to generate
        type: integer
+      params:
+        additionalProperties:
+          type: string
+        description: backend-specific generation parameters
+        type: object
      prompt:
        description: text description of the video to generate
        type: string
@@ -4723,7 +4731,7 @@ paths:
          description: Response
          schema:
            $ref: '#/definitions/schema.OpenAIResponse'
-      summary: Creates a video given a prompt.
+      summary: Creates a video from a prompt and optional image or audio conditioning.
      tags:
      - video
  /ws/backend-logs/{modelId}:
--- a/tests/e2e/distributed/distributed_full_flow_test.go
+++ b/tests/e2e/distributed/distributed_full_flow_test.go
@@ -40,6 +40,8 @@ type testLLM struct {
 	dstOutput []byte
 	// lastSrc records the last Src/input path seen (for verifying staging rewrote it).
 	lastSrc string
+	// lastVideoAudio records the staged Audio path from GenerateVideo.
+	lastVideoAudio string
 	// lastAudioDst records the Dst field from AudioTranscription (it's an input, not output).
 	lastAudioDst string
 	// lastTTSModel records the Model field from TTS requests (for verifying path rewriting).
@@ -69,6 +71,7 @@ func (t *testLLM) GenerateImage(req *pb.GenerateImageRequest) error {

 func (t *testLLM) GenerateVideo(req *pb.GenerateVideoRequest) error {
 	t.lastSrc = req.StartImage
+	t.lastVideoAudio = req.Audio
 	if req.Dst != "" && len(t.dstOutput) > 0 {
 		return os.WriteFile(req.Dst, t.dstOutput, 0644)
 	}
@@ -710,7 +713,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func()
 		Expect(retrievedData).To(Equal(outputData))
 	})

-	It("should round-trip output via FileStagingClient.GenerateVideo (StartImage + Dst)", func() {
+	It("should round-trip output via FileStagingClient.GenerateVideo (StartImage + Audio + Dst)", func() {
 		outputData := []byte("MP4 video generated by the backend")

 		llm := &testLLM{dstOutput: outputData}
@@ -723,6 +726,9 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func()
 		startImageContent := []byte("start frame image data")
 		startImagePath := filepath.Join(imgDir, "start.png")
 		Expect(os.WriteFile(startImagePath, startImageContent, 0644)).To(Succeed())
+		audioContent := []byte("audio conditioning data")
+		audioPath := filepath.Join(imgDir, "speech.wav")
+		Expect(os.WriteFile(audioPath, audioContent, 0644)).To(Succeed())

 		localOutputDir := GinkgoT().TempDir()
 		frontendDst := filepath.Join(localOutputDir, "generated.mp4")
@@ -730,6 +736,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func()
 		genResult, err := result.Client.GenerateVideo(ctx, &pb.GenerateVideoRequest{
 			Prompt:     "a flying cat",
 			StartImage: startImagePath,
+			Audio:      audioPath,
 			Dst:        frontendDst,
 			NumFrames:  16,
 		})
@@ -741,6 +748,10 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func()
 		stagedStartData, err := os.ReadFile(llm.lastSrc)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(stagedStartData).To(Equal(startImageContent))
+		Expect(llm.lastVideoAudio).To(ContainSubstring(stagingDir))
+		stagedAudioData, err := os.ReadFile(llm.lastVideoAudio)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(stagedAudioData).To(Equal(audioContent))

 		// Verify output: video was pulled back
 		retrievedData, err := os.ReadFile(frontendDst)