From b00422e45fcfff5abb67066c9313e2628aea0d44 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:58:46 +0200 Subject: [PATCH] feat(backends): add LongCat video and avatar generation (#10792) * feat(backends): add LongCat video and avatar generation Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] [web] * refactor(config): declare model I/O modalities Make model configs declare input and output modalities so capability discovery no longer branches on backend or checkpoint names. Complete the LongCat gallery and user documentation, make the SDPA patch apply to the pinned upstream revision, and stabilize the Agent Jobs race exposed by the required hook. Assisted-by: Codex:GPT-5 [web] --------- Co-authored-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 39 + .github/workflows/bump_deps.yaml | 4 + Makefile | 8 +- backend/README.md | 1 + backend/backend.proto | 5 +- backend/index.yaml | 62 ++ backend/python/README.md | 1 + backend/python/longcat-video/.gitignore | 6 + backend/python/longcat-video/Makefile | 36 + backend/python/longcat-video/README.md | 44 + backend/python/longcat-video/backend.py | 904 ++++++++++++++++++ backend/python/longcat-video/install.sh | 16 + backend/python/longcat-video/longcat_utils.py | 182 ++++ .../0001-sdpa-attention-fallback.patch | 75 ++ .../longcat-video/requirements-after.txt | 1 + .../python/longcat-video/requirements-cpu.txt | 3 + .../longcat-video/requirements-cublas12.txt | 3 + .../longcat-video/requirements-cublas13.txt | 3 + .../longcat-video/requirements-l4t13.txt | 3 + backend/python/longcat-video/requirements.txt | 23 + backend/python/longcat-video/run.sh | 12 + backend/python/longcat-video/test.py | 218 +++++ backend/python/longcat-video/test.sh | 12 + core/backend/video.go | 78 +- core/config/backend_capabilities.go | 10 +- core/config/meta/constants.go | 8 + core/config/meta/registry.go | 16 + core/config/model_capabilities.go | 98 +- core/config/model_capabilities_test.go | 27 + core/config/model_config.go | 6 +- core/gallery/importers/importers.go | 7 +- core/gallery/importers/importers_test.go | 12 +- core/gallery/importers/longcat-video.go | 127 +++ core/gallery/importers/longcat-video_test.go | 116 +++ core/http/app_test.go | 17 +- .../endpoints/localai/api_instructions.go | 3 +- core/http/endpoints/localai/video.go | 225 +++-- .../endpoints/localai/video_internal_test.go | 68 ++ .../e2e/import-form-ux-batch-e.spec.js | 3 +- core/http/react-ui/e2e/media-history.spec.js | 21 +- .../public/locales/de/importModel.json | 1 + .../react-ui/public/locales/de/media.json | 7 +- .../public/locales/en/importModel.json | 1 + .../react-ui/public/locales/en/media.json | 7 +- .../public/locales/es/importModel.json | 1 + .../react-ui/public/locales/es/media.json | 7 +- .../public/locales/id/importModel.json | 1 + .../react-ui/public/locales/id/media.json | 7 +- .../public/locales/it/importModel.json | 1 + .../react-ui/public/locales/it/media.json | 7 +- .../public/locales/ko/importModel.json | 1 + .../react-ui/public/locales/ko/media.json | 7 +- .../public/locales/zh-CN/importModel.json | 1 + .../react-ui/public/locales/zh-CN/media.json | 7 +- core/http/react-ui/src/App.css | 18 + .../react-ui/src/components/ModalityChips.jsx | 1 + core/http/react-ui/src/pages/ImportModel.jsx | 2 +- core/http/react-ui/src/pages/VideoGen.jsx | 51 +- core/schema/localai.go | 2 + core/services/nodes/file_staging_client.go | 9 +- docs/content/advanced/model-configuration.md | 17 + docs/content/features/_index.en.md | 3 +- docs/content/features/api-discovery.md | 2 +- docs/content/features/backends.md | 4 +- docs/content/features/longcat-video.md | 169 ++++ docs/content/features/video-generation.md | 9 +- docs/content/whats-new.md | 3 +- gallery/index.yaml | 76 ++ swagger/docs.go | 13 +- swagger/swagger.json | 13 +- swagger/swagger.yaml | 10 +- .../distributed/distributed_full_flow_test.go | 13 +- 72 files changed, 2784 insertions(+), 190 deletions(-) create mode 100644 backend/python/longcat-video/.gitignore create mode 100644 backend/python/longcat-video/Makefile create mode 100644 backend/python/longcat-video/README.md create mode 100755 backend/python/longcat-video/backend.py create mode 100755 backend/python/longcat-video/install.sh create mode 100644 backend/python/longcat-video/longcat_utils.py create mode 100644 backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch create mode 100644 backend/python/longcat-video/requirements-after.txt create mode 100644 backend/python/longcat-video/requirements-cpu.txt create mode 100644 backend/python/longcat-video/requirements-cublas12.txt create mode 100644 backend/python/longcat-video/requirements-cublas13.txt create mode 100644 backend/python/longcat-video/requirements-l4t13.txt create mode 100644 backend/python/longcat-video/requirements.txt create mode 100755 backend/python/longcat-video/run.sh create mode 100644 backend/python/longcat-video/test.py create mode 100755 backend/python/longcat-video/test.sh create mode 100644 core/gallery/importers/longcat-video.go create mode 100644 core/gallery/importers/longcat-video_test.go create mode 100644 core/http/endpoints/localai/video_internal_test.go create mode 100644 docs/content/features/longcat-video.md diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 4d1bd34fd..111c8b32b 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -478,6 +478,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-longcat-video' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1149,6 +1162,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-longcat-video' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1357,6 +1383,19 @@ include: backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" context: "./" + - build-type: 'l4t' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + ubuntu-version: '2404' + backend: "longcat-video" + dockerfile: "./backend/Dockerfile.python" + context: "./" - build-type: 'l4t' cuda-major-version: "13" cuda-minor-version: "0" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index cde01ef52..d9b3b4069 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -26,6 +26,10 @@ jobs: variable: "DS4_VERSION" branch: "main" file: "backend/cpp/ds4/Makefile" + - repository: "meituan-longcat/LongCat-Video" + variable: "LONGCAT_VIDEO_VERSION" + branch: "main" + file: "backend/python/longcat-video/Makefile" - repository: "localai-org/privacy-filter.cpp" variable: "PRIVACY_FILTER_VERSION" branch: "master" diff --git a/Makefile b/Makefile index c6f38b1d0..da2e638b0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -565,6 +565,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/chatterbox $(MAKE) -C backend/python/vllm $(MAKE) -C backend/python/vllm-omni + $(MAKE) -C backend/python/longcat-video $(MAKE) -C backend/python/sglang $(MAKE) -C backend/python/vibevoice $(MAKE) -C backend/python/liquid-audio @@ -594,6 +595,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/chatterbox test $(MAKE) -C backend/python/vllm test $(MAKE) -C backend/python/vllm-omni test + $(MAKE) -C backend/python/longcat-video test $(MAKE) -C backend/python/vibevoice test $(MAKE) -C backend/python/liquid-audio test $(MAKE) -C backend/python/moonshine test @@ -1254,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true BACKEND_KOKORO = kokoro|python|.|false|true BACKEND_VLLM = vllm|python|.|false|true BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true +BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true BACKEND_SGLANG = sglang|python|.|false|true BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true BACKEND_CHATTERBOX = chatterbox|python|.|false|true @@ -1339,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS))) $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO))) $(eval $(call generate-docker-build-target,$(BACKEND_VLLM))) $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI))) +$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO))) $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG))) $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS))) $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX))) @@ -1375,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter ######################################################## ### Mock Backend for E2E Tests diff --git a/backend/README.md b/backend/README.md index 10c01d524..0e92a0f03 100644 --- a/backend/README.md +++ b/backend/README.md @@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **vllm**: High-performance LLM inference - **mlx**: Apple Silicon optimization - **diffusers**: Stable Diffusion models +- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation - **Audio**: coqui, faster-whisper, kitten-tts - **Vision**: mlx-vlm, rfdetr - **Specialized**: rerankers, chatterbox, kokoro diff --git a/backend/backend.proto b/backend/backend.proto index 01c5b63a7..ad62c6df0 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -577,6 +577,10 @@ message GenerateVideoRequest { float cfg_scale = 10; // Classifier-free guidance scale int32 step = 11; // Number of inference steps string dst = 12; // Output path for the generated video + string audio = 13; // Path to staged audio for audio-conditioned video + // Backend-specific per-request generation parameters. Values are strings + // and are validated/coerced by the selected backend. + map params = 14; } message TTSRequest { @@ -1256,4 +1260,3 @@ message ForwardReply { repeated ForwardHeader headers = 2; bytes body_chunk = 3; } - diff --git a/backend/index.yaml b/backend/index.yaml index 4d17bcc62..3374a9d71 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -824,6 +824,30 @@ nvidia-cuda-12: "cuda12-vllm-omni" nvidia-cuda-13: "cuda13-vllm-omni" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-omni" +- &longcat-video + name: "longcat-video" + alias: "longcat-video" + license: mit + urls: + - https://github.com/meituan-longcat/LongCat-Video + tags: + - text-to-video + - image-to-video + - audio-to-video + - avatar-generation + - video-generation + - CUDA + icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg + description: | + LongCat-Video generation for text, image, and audio-conditioned avatars. + Supports LongCat-Video and LongCat-Video-Avatar-1.5, including multi-segment + talking-head continuation and an SDPA path for NVIDIA Blackwell ARM64 systems. + Requires Linux with an NVIDIA CUDA GPU; CPU, ROCm, and macOS are unsupported. + capabilities: + nvidia: "cuda12-longcat-video" + nvidia-cuda-12: "cuda12-longcat-video" + nvidia-cuda-13: "cuda13-longcat-video" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video" - &mlx name: "mlx" icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4 @@ -3605,6 +3629,44 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm-omni" mirrors: - localai/localai-backends:master-gpu-rocm-hipblas-vllm-omni +# longcat-video +- !!merge <<: *longcat-video + name: "longcat-video-development" + capabilities: + nvidia: "cuda12-longcat-video-development" + nvidia-cuda-12: "cuda12-longcat-video-development" + nvidia-cuda-13: "cuda13-longcat-video-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video-development" +- !!merge <<: *longcat-video + name: "cuda12-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-longcat-video" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-longcat-video" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-nvidia-l4t-arm64-longcat-video" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video +- !!merge <<: *longcat-video + name: "cuda12-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-longcat-video" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-longcat-video" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-longcat-video +- !!merge <<: *longcat-video + name: "cuda13-nvidia-l4t-arm64-longcat-video-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video # rfdetr - !!merge <<: *rfdetr name: "rfdetr-development" diff --git a/backend/python/README.md b/backend/python/README.md index 45ee0e69c..1864502f5 100644 --- a/backend/python/README.md +++ b/backend/python/README.md @@ -27,6 +27,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro ### Computer Vision - **diffusers** - Stable Diffusion and image generation +- **longcat-video** - CUDA video and speech-driven avatar generation with LongCat-Video - **mlx-vlm** - Vision-language models for Apple Silicon - **rfdetr** - Object detection models diff --git a/backend/python/longcat-video/.gitignore b/backend/python/longcat-video/.gitignore new file mode 100644 index 000000000..1ee12d070 --- /dev/null +++ b/backend/python/longcat-video/.gitignore @@ -0,0 +1,6 @@ +backend_pb2.py +backend_pb2_grpc.py +lib/ +python/ +sources/ +venv/ diff --git a/backend/python/longcat-video/Makefile b/backend/python/longcat-video/Makefile new file mode 100644 index 000000000..267ed8917 --- /dev/null +++ b/backend/python/longcat-video/Makefile @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT + +LONGCAT_VIDEO_VERSION?=6b3f4b8582a8bc3f20f795735f5383716c4ba794 +LONGCAT_VIDEO_REPO?=https://github.com/meituan-longcat/LongCat-Video +LONGCAT_SOURCE_STAMP=sources/LongCat-Video/.localai-$(LONGCAT_VIDEO_VERSION) + +.PHONY: all +all: $(LONGCAT_SOURCE_STAMP) + bash install.sh + +$(LONGCAT_SOURCE_STAMP): patches/0001-sdpa-attention-fallback.patch + rm -rf sources/LongCat-Video + mkdir -p sources/LongCat-Video + cd sources/LongCat-Video && git init -q && \ + git remote add origin $(LONGCAT_VIDEO_REPO) && \ + git fetch --depth 1 origin $(LONGCAT_VIDEO_VERSION) && \ + git checkout --detach FETCH_HEAD && \ + git apply ../../patches/0001-sdpa-attention-fallback.patch && \ + rm -rf .git && \ + touch .localai-$(LONGCAT_VIDEO_VERSION) + +.PHONY: run +run: all + bash run.sh + +.PHONY: test +test: all + bash test.sh + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2.py backend_pb2_grpc.py + +.PHONY: clean +clean: protogen-clean + rm -rf __pycache__ lib python sources venv diff --git a/backend/python/longcat-video/README.md b/backend/python/longcat-video/README.md new file mode 100644 index 000000000..821de0130 --- /dev/null +++ b/backend/python/longcat-video/README.md @@ -0,0 +1,44 @@ +# LongCat Video backend + +This backend serves Meituan's `LongCat-Video` and +`LongCat-Video-Avatar-1.5` checkpoints through LocalAI's `GenerateVideo` +RPC. It supports: + +- text-to-video and image-to-video with `LongCat-Video`; +- audio + text-to-avatar and portrait + audio-to-avatar with Avatar 1.5; +- multi-segment avatar continuation for speech longer than one segment; +- PyTorch SDPA when FlashAttention is unavailable, including CUDA 13 ARM64 + systems such as NVIDIA DGX Spark. + +Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the +LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md) +for Studio and API examples, hardware requirements, and manual configuration. + +The upstream source is pinned in `Makefile` and patched at build time. The +patch adds only the missing SDPA attention branches; model and source licenses +remain MIT. + +## Model options + +| Option | Default | Description | +| --- | --- | --- | +| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`. The packaged backend guarantees only `sdpa`. | +| `use_distill` | `true` for Avatar, `false` for base | Loads the checkpoint's fast distillation LoRA. | +| `use_int8` | `false` | Loads Avatar 1.5's INT8 DiT. BF16 has a lower load-time peak on unified-memory systems. | +| `base_model` | `meituan-longcat/LongCat-Video` | Base components used by Avatar 1.5. | +| `max_segments` | `8` | Maximum avatar continuation segments accepted per request. | +| `resolution` | `480p` | Image-conditioned generation resolution (`480p` or `720p`). | + +Per-request `params` may set `num_segments`, `audio_guidance_scale`, +`offload_kv_cache`, `ref_img_index`, `mask_frame_range`, and `resolution`. + +Gallery and imported configs declare `known_input_modalities` and +`known_output_modalities`. Keep those declarations in manual configs as well; +they let model discovery distinguish base image-conditioned video from Avatar +audio conditioning without inspecting the backend or checkpoint name. + +LongCat is CUDA-only and very large. Avatar 1.5 also loads tokenizer, +text-encoder, and VAE components from the base checkpoint. Keep ample unified +memory and storage available; no CPU or macOS backend image is published. The +initial backend supports one GPU per process; tensor parallel sizes above one +are rejected explicitly. diff --git a/backend/python/longcat-video/backend.py b/backend/python/longcat-video/backend.py new file mode 100755 index 000000000..62beb0a0e --- /dev/null +++ b/backend/python/longcat-video/backend.py @@ -0,0 +1,904 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT + +import argparse +import datetime +import gc +import math +import os +import signal +import subprocess +import sys +import tempfile +import traceback +from concurrent import futures + +import grpc + +import backend_pb2 +import backend_pb2_grpc + +from longcat_utils import ( + BASE_MODEL_ID, + MODEL_KIND_AVATAR, + MODEL_KIND_BASE, + attention_overrides, + avatar_segments_for_duration, + avatar_segments_for_frames, + classify_model, + normalize_model_source, + normalize_num_frames, + parse_options, + require_bool, + require_float, + require_int, + validate_dimensions, +) + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "sources", "LongCat-Video")) + +from grpc_auth import get_auth_interceptors + + +MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1")) + +DEFAULT_NEGATIVE_PROMPT = ( + "Close-up, bright tones, overexposed, static, blurred details, subtitles, " + "paintings, low quality, JPEG compression residue, ugly, incomplete, extra " + "fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, " + "misshapen limbs, fused fingers, still picture, messy background, three legs, " + "many people in the background, walking backwards" +) + +LOAD_OPTIONS = { + "attention_backend", + "base_model", + "max_segments", + "resolution", + "use_distill", + "use_int8", +} + +REQUEST_PARAMS = { + "audio_guidance_scale", + "mask_frame_range", + "num_segments", + "offload_kv_cache", + "ref_img_index", + "resolution", +} + +BASE_CHECKPOINT_PATTERNS = [ + "config.json", + "model_index.json", + "dit/**", + "lora/cfg_step_lora.safetensors", + "scheduler/**", + "text_encoder/**", + "tokenizer/**", + "vae/**", +] + +AVATAR_BASE_PATTERNS = [ + "config.json", + "model_index.json", + "text_encoder/**", + "tokenizer/**", + "vae/**", +] + +AVATAR_COMMON_PATTERNS = [ + "config.json", + "lora/dmd_lora.safetensors", + "model_index.json", + "scheduler/**", + "whisper-large-v3/config.json", + "whisper-large-v3/model.safetensors", + "whisper-large-v3/preprocessor_config.json", +] + + +class BackendServicer(backend_pb2_grpc.BackendServicer): + def __init__(self): + self.model_kind = None + self.pipeline = None + self.options = {} + self.device_index = 0 + self.cp_split_hw = None + self._dist_store_dir = None + + def Health(self, request, context): + return backend_pb2.Reply(message=b"OK") + + def LoadModel(self, request, context): + model = request.Model + if request.ModelFile and os.path.isdir(request.ModelFile): + model = request.ModelFile + + model_kind = classify_model(model) + if model_kind is None: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "longcat-video only accepts LongCat-Video or LongCat-Video-Avatar-1.5 checkpoints", + ) + + try: + options = parse_options(request.Options) + unknown = sorted(set(options) - LOAD_OPTIONS) + if unknown: + raise ValueError(f"unknown model option(s): {', '.join(unknown)}") + + self._import_torch() + if not self.torch.cuda.is_available(): + return self._fail( + context, + grpc.StatusCode.FAILED_PRECONDITION, + "longcat-video requires an NVIDIA CUDA GPU", + ) + if request.TensorParallelSize > 1: + return self._fail( + context, + grpc.StatusCode.UNIMPLEMENTED, + "longcat-video currently supports one GPU per backend process", + ) + self._import_runtime() + + attention_name = str(options.get("attention_backend", "sdpa")).lower() + attention_overrides(attention_name) + resolution = str(options.get("resolution", "480p")).lower() + if resolution not in {"480p", "720p"}: + raise ValueError("resolution must be 480p or 720p") + + use_distill_default = model_kind == MODEL_KIND_AVATAR + use_distill = require_bool( + options.get("use_distill", use_distill_default), + "use_distill", + ) + use_int8 = require_bool(options.get("use_int8", False), "use_int8") + if model_kind == MODEL_KIND_BASE and use_int8: + raise ValueError( + "use_int8 is supported only by LongCat-Video-Avatar-1.5" + ) + + self.options = { + **options, + "attention_backend": attention_name, + "resolution": resolution, + "use_distill": use_distill, + "use_int8": use_int8, + "max_segments": require_int( + options.get("max_segments", 8), + "max_segments", + minimum=1, + maximum=64, + ), + } + + self._release_model() + self._ensure_distributed() + if model_kind == MODEL_KIND_BASE: + self._load_base_model(model) + else: + self._load_avatar_model(model) + self.model_kind = model_kind + print( + f"Loaded {normalize_model_source(model)} as {model_kind} " + f"with attention_backend={attention_name}", + file=sys.stderr, + ) + return backend_pb2.Result(message="Model loaded successfully", success=True) + except ValueError as err: + self._release_model() + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err)) + except Exception as err: + self._release_model() + print(f"Error loading LongCat model: {err}", file=sys.stderr) + traceback.print_exc() + return self._fail( + context, + grpc.StatusCode.INTERNAL, + f"failed to load LongCat model: {err}", + ) + + def Free(self, request, context): + self._release_model() + return backend_pb2.Result(message="Model released", success=True) + + def GenerateVideo(self, request, context): + if self.pipeline is None or self.model_kind is None: + return self._fail( + context, + grpc.StatusCode.FAILED_PRECONDITION, + "model is not loaded", + ) + if not request.prompt.strip(): + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "prompt is required", + ) + if not request.dst: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "output destination is required", + ) + if request.end_image: + return self._fail( + context, + grpc.StatusCode.INVALID_ARGUMENT, + "longcat-video does not support end_image conditioning", + ) + + request_state = {"finished": False} + + def interrupt_if_cancelled(): + if not request_state["finished"] and self.pipeline is not None: + self.pipeline._interrupt = True + + try: + params = dict(request.params) + unknown = sorted(set(params) - REQUEST_PARAMS) + if unknown: + raise ValueError(f"unknown request param(s): {', '.join(unknown)}") + + os.makedirs(os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True) + if hasattr(context, "add_callback"): + context.add_callback(interrupt_if_cancelled) + + if request.start_image and not os.path.isfile(request.start_image): + raise ValueError("start_image is not a readable staged file") + if request.num_frames < 0: + raise ValueError("num_frames must not be negative") + + if self.model_kind == MODEL_KIND_BASE: + if request.audio: + raise ValueError( + "audio input requires a LongCat-Video-Avatar-1.5 model" + ) + self._generate_base(request, params) + else: + self._generate_avatar(request, params, context) + + return backend_pb2.Result( + message="Video generated successfully", success=True + ) + except ValueError as err: + return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err)) + except Exception as err: + print(f"Error generating LongCat video: {err}", file=sys.stderr) + traceback.print_exc() + return self._fail( + context, + grpc.StatusCode.INTERNAL, + f"LongCat video generation failed: {err}", + ) + finally: + request_state["finished"] = True + if self.pipeline is not None: + self.pipeline._interrupt = False + + def _import_torch(self): + if hasattr(self, "torch"): + return + + import torch + + self.torch = torch + + def _import_runtime(self): + if hasattr(self, "LongCatVideoPipeline"): + return + + import imageio.v2 as imageio + import imageio_ffmpeg + import librosa + import numpy as np + import torch.distributed as dist + from diffusers.utils import load_image + from huggingface_hub import snapshot_download + from PIL import Image + from transformers import AutoTokenizer, UMT5EncoderModel + + from longcat_video.audio_process import ( + get_audio_encoder, + get_audio_feature_extractor, + ) + from longcat_video.context_parallel import context_parallel_util + from longcat_video.modules.autoencoder_kl_wan import AutoencoderKLWan + from longcat_video.modules.avatar.longcat_video_dit_avatar import ( + LongCatVideoAvatarTransformer3DModel, + ) + from longcat_video.modules.longcat_video_dit import ( + LongCatVideoTransformer3DModel, + ) + from longcat_video.modules.quantization import load_quantized_dit + from longcat_video.modules.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + ) + from longcat_video.pipeline_longcat_video import LongCatVideoPipeline + from longcat_video.pipeline_longcat_video_avatar import ( + LongCatVideoAvatarPipeline, + ) + + self.imageio = imageio + self.imageio_ffmpeg = imageio_ffmpeg + self.librosa = librosa + self.np = np + self.dist = dist + self.load_image = load_image + self.snapshot_download = snapshot_download + self.Image = Image + self.AutoTokenizer = AutoTokenizer + self.UMT5EncoderModel = UMT5EncoderModel + self.get_audio_encoder = get_audio_encoder + self.get_audio_feature_extractor = get_audio_feature_extractor + self.context_parallel_util = context_parallel_util + self.AutoencoderKLWan = AutoencoderKLWan + self.LongCatVideoAvatarTransformer3DModel = LongCatVideoAvatarTransformer3DModel + self.LongCatVideoTransformer3DModel = LongCatVideoTransformer3DModel + self.load_quantized_dit = load_quantized_dit + self.FlowMatchEulerDiscreteScheduler = FlowMatchEulerDiscreteScheduler + self.LongCatVideoPipeline = LongCatVideoPipeline + self.LongCatVideoAvatarPipeline = LongCatVideoAvatarPipeline + + def _ensure_distributed(self): + self.torch.cuda.set_device(self.device_index) + if not self.dist.is_initialized(): + self._dist_store_dir = tempfile.mkdtemp(prefix="localai-longcat-dist-") + init_file = os.path.join(self._dist_store_dir, "store") + self.dist.init_process_group( + backend="nccl", + init_method=f"file://{init_file}", + rank=0, + world_size=1, + timeout=datetime.timedelta(hours=24), + ) + self.context_parallel_util.init_context_parallel( + context_parallel_size=1, + global_rank=0, + world_size=1, + ) + self.cp_split_hw = self.context_parallel_util.get_optimal_split(1) + + def _resolve_checkpoint(self, model, patterns): + source = normalize_model_source(model) + if os.path.isdir(source): + return source + print(f"Downloading required files for {source}", file=sys.stderr) + return self.snapshot_download(repo_id=source, allow_patterns=patterns) + + def _load_base_model(self, model): + checkpoint = self._resolve_checkpoint(model, BASE_CHECKPOINT_PATTERNS) + dtype = self.torch.bfloat16 + overrides = attention_overrides(self.options["attention_backend"]) + + tokenizer = self.AutoTokenizer.from_pretrained( + checkpoint, + subfolder="tokenizer", + ) + text_encoder = self.UMT5EncoderModel.from_pretrained( + checkpoint, + subfolder="text_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + vae = self.AutoencoderKLWan.from_pretrained( + checkpoint, + subfolder="vae", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained( + checkpoint, + subfolder="scheduler", + ) + dit = self.LongCatVideoTransformer3DModel.from_pretrained( + checkpoint, + subfolder="dit", + cp_split_hw=self.cp_split_hw, + torch_dtype=dtype, + low_cpu_mem_usage=True, + **overrides, + ) + if self.options["use_distill"]: + dit.load_lora( + os.path.join(checkpoint, "lora", "cfg_step_lora.safetensors"), + "cfg_step_lora", + ) + dit.enable_loras(["cfg_step_lora"]) + + self.pipeline = self.LongCatVideoPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + scheduler=scheduler, + dit=dit, + ) + self.pipeline.to(self.device_index) + + def _load_avatar_model(self, model): + avatar_patterns = list(AVATAR_COMMON_PATTERNS) + model_subfolder = ( + "base_model_int8" if self.options["use_int8"] else "base_model" + ) + avatar_patterns.append(f"{model_subfolder}/**") + checkpoint = self._resolve_checkpoint(model, avatar_patterns) + + base_model = self.options.get("base_model") + if not base_model and os.path.isdir(normalize_model_source(model)): + sibling = os.path.join( + os.path.dirname(normalize_model_source(model)), "LongCat-Video" + ) + if os.path.isdir(sibling): + base_model = sibling + base_model = base_model or BASE_MODEL_ID + if classify_model(str(base_model)) != MODEL_KIND_BASE: + raise ValueError("base_model must point to a LongCat-Video checkpoint") + base_checkpoint = self._resolve_checkpoint(base_model, AVATAR_BASE_PATTERNS) + + dtype = self.torch.bfloat16 + overrides = attention_overrides(self.options["attention_backend"]) + tokenizer = self.AutoTokenizer.from_pretrained( + base_checkpoint, + subfolder="tokenizer", + ) + text_encoder = self.UMT5EncoderModel.from_pretrained( + base_checkpoint, + subfolder="text_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + vae = self.AutoencoderKLWan.from_pretrained( + base_checkpoint, + subfolder="vae", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained( + checkpoint, + subfolder="scheduler", + ) + + if self.options["use_int8"]: + previous_dtype = self.torch.get_default_dtype() + self.torch.set_default_dtype(dtype) + try: + dit = self.load_quantized_dit( + checkpoint, + subfolder="base_model_int8", + cp_split_hw=self.cp_split_hw, + **overrides, + ) + finally: + self.torch.set_default_dtype(previous_dtype) + else: + dit = self.LongCatVideoAvatarTransformer3DModel.from_pretrained( + checkpoint, + subfolder="base_model", + cp_split_hw=self.cp_split_hw, + torch_dtype=dtype, + low_cpu_mem_usage=True, + **overrides, + ) + + if self.options["use_distill"]: + dit.load_lora( + os.path.join(checkpoint, "lora", "dmd_lora.safetensors"), + "dmd", + multiplier=1.0, + lora_network_dim=128, + lora_network_alpha=64, + ) + dit.enable_loras(["dmd"]) + + audio_checkpoint = os.path.join(checkpoint, "whisper-large-v3") + audio_encoder = self.get_audio_encoder( + audio_checkpoint, + MODEL_KIND_AVATAR + "-v1.5", + ).to(self.device_index) + audio_feature_extractor = self.get_audio_feature_extractor( + audio_checkpoint, + MODEL_KIND_AVATAR + "-v1.5", + ) + self.pipeline = self.LongCatVideoAvatarPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + scheduler=scheduler, + dit=dit, + audio_encoder=audio_encoder, + audio_feature_extractor=audio_feature_extractor, + model_type="avatar-v1.5", + ) + self.pipeline.to(self.device_index) + + def _generate_base(self, request, params): + use_distill = self.options["use_distill"] + frames = normalize_num_frames(request.num_frames) + steps = ( + 16 + if use_distill + else require_int( + request.step or 50, + "step", + minimum=1, + maximum=200, + ) + ) + guidance_scale = ( + 1.0 + if use_distill + else require_float( + request.cfg_scale or 4.0, + "cfg_scale", + minimum=0.0, + maximum=30.0, + ) + ) + fps = require_int(request.fps or 15, "fps", minimum=1, maximum=60) + seed = request.seed if request.seed > 0 else 42 + negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT + generator = self.torch.Generator(device=self.device_index).manual_seed(seed) + + if request.start_image: + resolution = self._resolution(params) + image = self.load_image(request.start_image) + output = self.pipeline.generate_i2v( + image=image, + prompt=request.prompt, + negative_prompt=negative_prompt, + resolution=resolution, + num_frames=frames, + num_inference_steps=steps, + use_distill=use_distill, + guidance_scale=guidance_scale, + generator=generator, + )[0] + else: + width, height = validate_dimensions(request.width, request.height) + output = self.pipeline.generate_t2v( + prompt=request.prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=frames, + num_inference_steps=steps, + use_distill=use_distill, + guidance_scale=guidance_scale, + generator=generator, + )[0] + + self._save_video(output, request.dst, fps) + + def _generate_avatar(self, request, params, context): + if not request.audio: + raise ValueError("audio is required for LongCat-Video-Avatar-1.5") + if not os.path.isfile(request.audio): + raise ValueError("audio input is not a readable staged file") + + use_distill = self.options["use_distill"] + steps = ( + 8 + if use_distill + else require_int( + request.step or 50, + "step", + minimum=1, + maximum=200, + ) + ) + text_guidance = ( + 1.0 + if use_distill + else require_float( + request.cfg_scale or 4.0, + "cfg_scale", + minimum=0.0, + maximum=30.0, + ) + ) + audio_guidance = ( + 1.0 + if use_distill + else require_float( + params.get("audio_guidance_scale", 4.0), + "audio_guidance_scale", + minimum=0.0, + maximum=20.0, + ) + ) + seed = request.seed if request.seed > 0 else 42 + generator = self.torch.Generator(device=self.device_index).manual_seed(seed) + negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT + resolution = self._resolution(params) + + speech, sample_rate = self.librosa.load(request.audio, sr=16000, mono=True) + if speech.size == 0: + raise ValueError("audio contains no samples") + audio_duration = len(speech) / sample_rate + segments = self._avatar_segments(request, params, audio_duration) + + segment_frames = 93 + conditioning_frames = 13 + avatar_fps = 25 + generated_duration = ( + segment_frames + (segments - 1) * (segment_frames - conditioning_frames) + ) / avatar_fps + pad_samples = max( + 0, math.ceil((generated_duration - audio_duration) * sample_rate) + ) + if pad_samples: + speech = self.np.pad(speech, (0, pad_samples)) + + full_audio_embedding = self.pipeline.get_audio_embedding( + speech, + fps=avatar_fps, + device=self.device_index, + sample_rate=sample_rate, + model_type="avatar-v1.5", + ) + if not self.torch.isfinite(full_audio_embedding).all(): + raise ValueError("audio encoder returned non-finite values") + + indices = self.torch.arange(5) - 2 + + def audio_window(start_index): + centers = self.torch.arange( + start_index, + start_index + segment_frames, + ).unsqueeze(1) + indices.unsqueeze(0) + centers = self.torch.clamp( + centers, + min=0, + max=full_audio_embedding.shape[0] - 1, + ) + return full_audio_embedding[centers][None, ...].to(self.device_index) + + audio_start = 0 + common = { + "prompt": request.prompt, + "negative_prompt": negative_prompt, + "num_frames": segment_frames, + "num_inference_steps": steps, + "text_guidance_scale": text_guidance, + "audio_guidance_scale": audio_guidance, + "output_type": "both", + "generator": generator, + "audio_emb": audio_window(audio_start), + "use_distill": use_distill, + } + + if request.start_image: + output, latent = self.pipeline.generate_ai2v( + image=self.load_image(request.start_image), + resolution=resolution, + **common, + ) + else: + width, height = validate_dimensions(request.width, request.height) + output, latent = self.pipeline.generate_at2v( + height=height, + width=width, + **common, + ) + + video = self._frames_to_pil(output[0]) + width, height = video[0].size + current_video = video + reference_latent = latent[:, :, :1].clone() + all_frames = list(video) + + for segment in range(1, segments): + if hasattr(context, "is_active") and not context.is_active(): + raise RuntimeError("request was cancelled") + print( + f"Generating avatar segment {segment + 1}/{segments}", file=sys.stderr + ) + audio_start += segment_frames - conditioning_frames + output, latent = self.pipeline.generate_avc( + video=current_video, + video_latent=latent, + prompt=request.prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=segment_frames, + num_cond_frames=conditioning_frames, + num_inference_steps=steps, + text_guidance_scale=text_guidance, + audio_guidance_scale=audio_guidance, + generator=generator, + output_type="both", + use_kv_cache=True, + offload_kv_cache=require_bool( + params.get("offload_kv_cache", False), + "offload_kv_cache", + ), + enhance_hf=not use_distill, + audio_emb=audio_window(audio_start), + ref_latent=reference_latent, + ref_img_index=require_int( + params.get("ref_img_index", 10), + "ref_img_index", + minimum=-30, + maximum=30, + ), + mask_frame_range=require_int( + params.get("mask_frame_range", 3), + "mask_frame_range", + minimum=0, + maximum=32, + ), + use_distill=use_distill, + ) + current_video = self._frames_to_pil(output[0]) + all_frames.extend(current_video[conditioning_frames:]) + + self._save_avatar_video(all_frames, request.audio, request.dst, avatar_fps) + + def _avatar_segments(self, request, params, audio_duration): + if "num_segments" in params: + segments = require_int( + params["num_segments"], + "num_segments", + minimum=1, + ) + elif request.num_frames > 0: + segments = avatar_segments_for_frames(request.num_frames) + else: + segments = avatar_segments_for_duration(audio_duration) + + max_segments = self.options["max_segments"] + if segments > max_segments: + raise ValueError( + f"request needs {segments} avatar segments, but max_segments is {max_segments}; " + "trim the audio or raise the model's max_segments option" + ) + return segments + + def _resolution(self, params): + resolution = str(params.get("resolution", self.options["resolution"])).lower() + if resolution not in {"480p", "720p"}: + raise ValueError("resolution must be 480p or 720p") + return resolution + + def _frames_to_pil(self, frames): + images = [] + for frame in frames: + array = self.np.asarray(frame) + if self.np.issubdtype(array.dtype, self.np.floating): + array = self.np.clip(array, 0.0, 1.0) * 255 + images.append(self.Image.fromarray(array.astype(self.np.uint8))) + return images + + def _save_video(self, frames, path, fps): + writer = self.imageio.get_writer( + path, + format="FFMPEG", + mode="I", + fps=fps, + codec="libx264", + macro_block_size=1, + ffmpeg_params=[ + "-crf", + "18", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + "-f", + "mp4", + ], + ) + try: + for frame in frames: + array = self.np.asarray(frame) + if self.np.issubdtype(array.dtype, self.np.floating): + array = self.np.clip(array, 0.0, 1.0) * 255 + writer.append_data(array.astype(self.np.uint8)) + finally: + writer.close() + + def _save_avatar_video(self, frames, audio_path, dst, fps): + output_dir = os.path.dirname(dst) or "." + handle, silent_path = tempfile.mkstemp( + prefix="longcat-silent-", + suffix=".mp4", + dir=output_dir, + ) + os.close(handle) + try: + self._save_video(frames, silent_path, fps) + command = [ + self.imageio_ffmpeg.get_ffmpeg_exe(), + "-y", + "-i", + silent_path, + "-i", + audio_path, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + "192k", + "-shortest", + "-movflags", + "+faststart", + "-f", + "mp4", + dst, + ] + subprocess.run( + command, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as err: + details = (err.stderr or "ffmpeg failed")[-2000:] + raise RuntimeError(f"failed to mux avatar audio: {details}") from err + finally: + try: + os.remove(silent_path) + except FileNotFoundError: + pass + + def _release_model(self): + self.pipeline = None + self.model_kind = None + gc.collect() + if hasattr(self, "torch") and self.torch.cuda.is_available(): + self.torch.cuda.empty_cache() + self.torch.cuda.ipc_collect() + + @staticmethod + def _fail(context, code, message): + if context is not None: + context.set_code(code) + context.set_details(message) + return backend_pb2.Result(message=message, success=False) + + +def serve(address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ("grpc.max_message_length", 64 * 1024 * 1024), + ("grpc.max_send_message_length", 64 * 1024 * 1024), + ("grpc.max_receive_message_length", 64 * 1024 * 1024), + ], + interceptors=get_auth_interceptors(), + ) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print(f"LongCat Video backend listening on {address}", file=sys.stderr) + + def stop_server(signum, frame): + del signum, frame + server.stop(0) + + signal.signal(signal.SIGINT, stop_server) + signal.signal(signal.SIGTERM, stop_server) + server.wait_for_termination() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the LongCat Video gRPC backend") + parser.add_argument( + "--addr", + default="localhost:50051", + help="address on which to serve the backend", + ) + arguments = parser.parse_args() + serve(arguments.addr) diff --git a/backend/python/longcat-video/install.sh b/backend/python/longcat-video/install.sh new file mode 100755 index 000000000..4b0b2d00a --- /dev/null +++ b/backend/python/longcat-video/install.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +PYTHON_VERSION="3.12" +PYTHON_PATCH="12" +PY_STANDALONE_TAG="20251120" + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +installRequirements diff --git a/backend/python/longcat-video/longcat_utils.py b/backend/python/longcat-video/longcat_utils.py new file mode 100644 index 000000000..3d4ac1040 --- /dev/null +++ b/backend/python/longcat-video/longcat_utils.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: MIT + +import json +import math +import os +from urllib.parse import urlparse + + +BASE_MODEL_ID = "meituan-longcat/LongCat-Video" +AVATAR_MODEL_ID = "meituan-longcat/LongCat-Video-Avatar-1.5" +MODEL_KIND_BASE = "base" +MODEL_KIND_AVATAR = "avatar" + +ATTENTION_OVERRIDES = { + "auto": {}, + "sdpa": { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": False, + }, + "flash2": { + "enable_flashattn2": True, + "enable_flashattn3": False, + "enable_xformers": False, + }, + "flash3": { + "enable_flashattn2": False, + "enable_flashattn3": True, + "enable_xformers": False, + }, + "xformers": { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": True, + }, +} + + +def parse_options(values): + options = {} + for raw in values: + if ":" not in raw: + options[raw.strip()] = True + continue + key, value = raw.split(":", 1) + key = key.strip() + value = value.strip() + if not key: + continue + lower = value.lower() + if lower in {"true", "false"}: + options[key] = lower == "true" + continue + try: + options[key] = int(value) + continue + except ValueError: + pass + try: + options[key] = float(value) + continue + except ValueError: + pass + options[key] = value + return options + + +def require_bool(value, name): + if isinstance(value, bool): + return value + if isinstance(value, str) and value.lower() in {"true", "false"}: + return value.lower() == "true" + raise ValueError(f"{name} must be true or false") + + +def require_int(value, name, minimum=None, maximum=None): + try: + parsed = int(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be an integer") from err + if minimum is not None and parsed < minimum: + raise ValueError(f"{name} must be at least {minimum}") + if maximum is not None and parsed > maximum: + raise ValueError(f"{name} must be at most {maximum}") + return parsed + + +def require_float(value, name, minimum=None, maximum=None): + try: + parsed = float(value) + except (TypeError, ValueError) as err: + raise ValueError(f"{name} must be a number") from err + if minimum is not None and parsed < minimum: + raise ValueError(f"{name} must be at least {minimum}") + if maximum is not None and parsed > maximum: + raise ValueError(f"{name} must be at most {maximum}") + return parsed + + +def attention_overrides(name): + try: + return dict(ATTENTION_OVERRIDES[name]) + except KeyError as err: + choices = ", ".join(ATTENTION_OVERRIDES) + raise ValueError(f"attention_backend must be one of: {choices}") from err + + +def _model_name_from_directory(path): + for filename in ("model_index.json", "config.json"): + config_path = os.path.join(path, filename) + try: + with open(config_path, "r", encoding="utf-8") as config_file: + model_name = json.load(config_file).get("model_name", "") + except (FileNotFoundError, OSError, ValueError, TypeError): + continue + if model_name: + return model_name + return "" + + +def normalize_model_source(model): + value = model.rstrip("/") + for prefix in ("huggingface://", "hf://"): + if value.startswith(prefix): + return value[len(prefix) :] + parsed = urlparse(value) + if parsed.scheme in {"http", "https"} and parsed.netloc.lower() == "huggingface.co": + parts = [part for part in parsed.path.split("/") if part] + if len(parts) >= 2: + return "/".join(parts[:2]) + return value + + +def classify_model(model): + if not model: + return None + normalized = normalize_model_source(model) + if os.path.isdir(normalized): + name = _model_name_from_directory(normalized).lower() + if name == "longcat-video": + return MODEL_KIND_BASE + if name == "longcat-video-avatar-1.5": + return MODEL_KIND_AVATAR + return None + + normalized = normalized.lower() + if normalized == BASE_MODEL_ID.lower(): + return MODEL_KIND_BASE + if normalized == AVATAR_MODEL_ID.lower(): + return MODEL_KIND_AVATAR + return None + + +def normalize_num_frames(value, default=93): + frames = default if not value or value < 1 else value + return max(1, ((frames - 1) // 4) * 4 + 1) + + +def avatar_segments_for_frames(frames): + if not frames or frames <= 93: + return 1 + return 1 + math.ceil((frames - 93) / 80) + + +def avatar_segments_for_duration(duration_seconds, fps=25): + if duration_seconds <= 0: + return 1 + return avatar_segments_for_frames(math.ceil(duration_seconds * fps)) + + +def validate_dimensions(width, height): + width = width or 832 + height = height or 480 + if width < 256 or height < 256: + raise ValueError("width and height must each be at least 256") + if width > 1280 or height > 768: + raise ValueError("width and height must not exceed 1280x768") + if width % 16 != 0 or height % 16 != 0: + raise ValueError("width and height must be divisible by 16") + if width * height > 1280 * 768: + raise ValueError("requested video dimensions exceed the 1280x768 pixel limit") + return width, height diff --git a/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch b/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch new file mode 100644 index 000000000..76829351f --- /dev/null +++ b/backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch @@ -0,0 +1,75 @@ +diff --git a/longcat_video/modules/attention.py b/longcat_video/modules/attention.py +index bb5630f..9b9f3cc 100644 +--- a/longcat_video/modules/attention.py ++++ b/longcat_video/modules/attention.py +@@ -2,6 +2,7 @@ from typing import List, Optional + + import torch + import torch.nn as nn ++import torch.nn.functional as F + + from einops import rearrange + +@@ -100,7 +101,8 @@ class Attention(nn.Module): + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,) + x = rearrange(x, "B M H K -> B H M K") + else: +- raise RuntimeError("Unsupported attention operations.") ++ # Keep a dependency-free path for systems without optional kernels. ++ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + + return x + +@@ -245,8 +247,22 @@ class MultiHeadCrossAttention(nn.Module): + attn_bias = xformers.ops.fmha.attn_bias.BlockDiagonalMask.from_seqlens([N] * B, kv_seqlen) + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=attn_bias) + else: +- raise RuntimeError("Unsupported attention operations.") +- ++ # Preserve the variable-length block boundaries without materializing ++ # a dense attention mask. ++ blocks = [] ++ offset = 0 ++ for batch_index, key_count in enumerate(kv_seqlen): ++ query = q[0][batch_index * N:(batch_index + 1) * N] ++ key = k[0][offset:offset + key_count] ++ value = v[0][offset:offset + key_count] ++ output = F.scaled_dot_product_attention( ++ query.transpose(0, 1), ++ key.transpose(0, 1), ++ value.transpose(0, 1), ++ ) ++ blocks.append(output.transpose(0, 1)) ++ offset += key_count ++ x = torch.cat(blocks, dim=0) + + x = x.view(B, -1, C) + x = self.proj(x) +diff --git a/longcat_video/modules/avatar/attention.py b/longcat_video/modules/avatar/attention.py +index a169a7a..df9a469 100644 +--- a/longcat_video/modules/avatar/attention.py ++++ b/longcat_video/modules/avatar/attention.py +@@ -2,6 +2,7 @@ from typing import List, Optional + + import torch + import torch.nn as nn ++import torch.nn.functional as F + + from einops import rearrange + +@@ -111,7 +112,8 @@ class Attention(nn.Module): + x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,) + x = rearrange(x, "B M H K -> B H M K") + else: +- raise RuntimeError("Unsupported attention operations.") ++ # Keep a dependency-free path for systems without optional kernels. ++ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + + return x + +@@ -429,2 +431,5 @@ class SingleStreamAttention(nn.Module): ++ else: ++ # This branch uses the native PyTorch kernel when optional kernels are off. ++ x = F.scaled_dot_product_attention(q, encoder_k, encoder_v, scale=self.scale) + + # linear transform diff --git a/backend/python/longcat-video/requirements-after.txt b/backend/python/longcat-video/requirements-after.txt new file mode 100644 index 000000000..a9368375b --- /dev/null +++ b/backend/python/longcat-video/requirements-after.txt @@ -0,0 +1 @@ +accelerate diff --git a/backend/python/longcat-video/requirements-cpu.txt b/backend/python/longcat-video/requirements-cpu.txt new file mode 100644 index 000000000..01311205e --- /dev/null +++ b/backend/python/longcat-video/requirements-cpu.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cpu +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-cublas12.txt b/backend/python/longcat-video/requirements-cublas12.txt new file mode 100644 index 000000000..b3b28a042 --- /dev/null +++ b/backend/python/longcat-video/requirements-cublas12.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu126 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-cublas13.txt b/backend/python/longcat-video/requirements-cublas13.txt new file mode 100644 index 000000000..c3fcbcaec --- /dev/null +++ b/backend/python/longcat-video/requirements-cublas13.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu130 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements-l4t13.txt b/backend/python/longcat-video/requirements-l4t13.txt new file mode 100644 index 000000000..c3fcbcaec --- /dev/null +++ b/backend/python/longcat-video/requirements-l4t13.txt @@ -0,0 +1,3 @@ +--index-url https://download.pytorch.org/whl/cu130 +torch==2.12.1 +torchvision==0.27.1 diff --git a/backend/python/longcat-video/requirements.txt b/backend/python/longcat-video/requirements.txt new file mode 100644 index 000000000..e3ad4e3d6 --- /dev/null +++ b/backend/python/longcat-video/requirements.txt @@ -0,0 +1,23 @@ +certifi +diffusers==0.35.1 +einops==0.8.0 +ftfy==6.2.0 +grpcio==1.76.0 +huggingface-hub>=0.23,<1.0 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +librosa==0.11.0 +loguru==0.7.2 +numpy==1.26.4 +packaging +pillow +protobuf +pyloudnorm==0.1.1 +regex +safetensors +scipy==1.15.3 +sentencepiece +soundfile==0.13.1 +soxr==0.5.0.post1 +tqdm +transformers==4.41.0 diff --git a/backend/python/longcat-video/run.sh b/backend/python/longcat-video/run.sh new file mode 100755 index 000000000..eaf36bda0 --- /dev/null +++ b/backend/python/longcat-video/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +startBackend "$@" diff --git a/backend/python/longcat-video/test.py b/backend/python/longcat-video/test.py new file mode 100644 index 000000000..13f32b8e5 --- /dev/null +++ b/backend/python/longcat-video/test.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: MIT + +import importlib.util +import json +import os +import sys +import tempfile +import unittest + +BACKEND_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, BACKEND_DIR) + +# longcat-video is a backend directory, not an importable Python package name. +from longcat_utils import ( # noqa: E402 + MODEL_KIND_AVATAR, + MODEL_KIND_BASE, + attention_overrides, + avatar_segments_for_duration, + avatar_segments_for_frames, + classify_model, + normalize_model_source, + normalize_num_frames, + parse_options, + validate_dimensions, +) + + +SOURCE_DIR = os.path.join(BACKEND_DIR, "sources", "LongCat-Video") +try: + import torch + + sys.path.insert(0, SOURCE_DIR) + ATTENTION_TESTS_AVAILABLE = ( + os.path.isdir(SOURCE_DIR) and importlib.util.find_spec("triton") is not None + ) +except ImportError: + torch = None + ATTENTION_TESTS_AVAILABLE = False + +AVATAR_ATTENTION_TESTS_AVAILABLE = ATTENTION_TESTS_AVAILABLE and all( + importlib.util.find_spec(module) is not None + for module in ("pyloudnorm", "scipy", "torchvision") +) + + +class LongCatUtilsTest(unittest.TestCase): + def test_parse_options_preserves_colons_and_coerces_scalars(self): + options = parse_options( + [ + "use_distill:true", + "max_segments:4", + "audio_guidance_scale:3.5", + "source:https://example.com/model", + "flag", + ] + ) + + self.assertEqual(options["use_distill"], True) + self.assertEqual(options["max_segments"], 4) + self.assertEqual(options["audio_guidance_scale"], 3.5) + self.assertEqual(options["source"], "https://example.com/model") + self.assertEqual(options["flag"], True) + + def test_classify_model_accepts_only_supported_longcat_models(self): + cases = { + "meituan-longcat/LongCat-Video": MODEL_KIND_BASE, + "https://huggingface.co/meituan-longcat/LongCat-Video": MODEL_KIND_BASE, + "hf://meituan-longcat/LongCat-Video-Avatar-1.5": MODEL_KIND_AVATAR, + "other-org/LongCat-Video": None, + "meituan-longcat/LongCat-Video-Avatar": None, + "some-org/unrelated-model": None, + } + + for model, expected in cases.items(): + with self.subTest(model=model): + self.assertEqual(classify_model(model), expected) + + def test_classify_model_reads_local_checkpoint_metadata(self): + with tempfile.TemporaryDirectory() as directory: + with open( + os.path.join(directory, "model_index.json"), + "w", + encoding="utf-8", + ) as config_file: + json.dump({"model_name": "LongCat-Video-Avatar-1.5"}, config_file) + + self.assertEqual(classify_model(directory), MODEL_KIND_AVATAR) + + def test_normalize_model_source_handles_huggingface_uri_forms(self): + self.assertEqual( + normalize_model_source( + "https://huggingface.co/meituan-longcat/LongCat-Video/tree/main" + ), + "meituan-longcat/LongCat-Video", + ) + self.assertEqual( + normalize_model_source("huggingface://meituan-longcat/LongCat-Video"), + "meituan-longcat/LongCat-Video", + ) + + def test_frame_and_segment_rounding_matches_longcat_temporal_shape(self): + self.assertEqual(normalize_num_frames(94), 93) + self.assertEqual(normalize_num_frames(0), 93) + self.assertEqual(avatar_segments_for_frames(93), 1) + self.assertEqual(avatar_segments_for_frames(94), 2) + self.assertEqual(avatar_segments_for_frames(173), 2) + self.assertEqual(avatar_segments_for_frames(174), 3) + self.assertEqual(avatar_segments_for_duration(10.0), 3) + + def test_dimensions_are_bounded_and_aligned(self): + self.assertEqual(validate_dimensions(0, 0), (832, 480)) + self.assertEqual(validate_dimensions(512, 512), (512, 512)) + with self.assertRaisesRegex(ValueError, "divisible by 16"): + validate_dimensions(513, 512) + with self.assertRaisesRegex(ValueError, "must not exceed"): + validate_dimensions(1920, 1080) + + def test_attention_backend_validation(self): + self.assertEqual( + attention_overrides("sdpa"), + { + "enable_flashattn2": False, + "enable_flashattn3": False, + "enable_xformers": False, + }, + ) + with self.assertRaisesRegex(ValueError, "attention_backend"): + attention_overrides("unknown") + + +@unittest.skipUnless( + ATTENTION_TESTS_AVAILABLE, + "patched LongCat source and torch are required for attention tests", +) +class SDPAFallbackTest(unittest.TestCase): + def test_base_self_attention_matches_reference(self): + from longcat_video.modules.attention import Attention + + dim, heads, sequence = 64, 4, 32 + attention = Attention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + enable_bsa=False, + ).float() + query = torch.randn(2, heads, sequence, dim // heads) + key = torch.randn_like(query) + value = torch.randn_like(query) + + output = attention._process_attn(query, key, value, shape=(1, 1, sequence)) + reference = ( + torch.softmax( + (query @ key.transpose(-1, -2)) * attention.scale, + dim=-1, + ) + @ value + ) + + self.assertLess((output - reference).abs().max().item(), 1e-4) + + @unittest.skipUnless( + AVATAR_ATTENTION_TESTS_AVAILABLE, + "avatar audio dependencies are required for the avatar attention test", + ) + def test_avatar_self_attention_matches_reference(self): + from longcat_video.modules.avatar.attention import Attention + + dim, heads, sequence = 64, 4, 16 + attention = Attention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + ).float() + query = torch.randn(1, heads, sequence, dim // heads) + key = torch.randn_like(query) + value = torch.randn_like(query) + + output = attention._process_attn(query, key, value, shape=(1, 1, sequence)) + reference = ( + torch.softmax( + (query @ key.transpose(-1, -2)) * attention.scale, + dim=-1, + ) + @ value + ) + + self.assertLess((output - reference).abs().max().item(), 1e-4) + + def test_base_cross_attention_remains_block_diagonal(self): + from longcat_video.modules.attention import MultiHeadCrossAttention + + dim, heads = 64, 4 + attention = MultiHeadCrossAttention( + dim, + heads, + enable_flashattn2=False, + enable_flashattn3=False, + enable_xformers=False, + ).float() + query = torch.randn(2, 8, dim) + key_lengths = [5, 7] + condition = torch.randn(1, sum(key_lengths), dim) + + first = attention._process_cross_attn(query, condition, key_lengths) + changed = condition.clone() + changed[:, key_lengths[0] :] = torch.randn_like(changed[:, key_lengths[0] :]) + second = attention._process_cross_attn(query, changed, key_lengths) + + self.assertLess((first[0] - second[0]).abs().max().item(), 1e-5) + self.assertGreater((first[1] - second[1]).abs().max().item(), 1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/longcat-video/test.sh b/backend/python/longcat-video/test.sh new file mode 100755 index 000000000..59cb0da32 --- /dev/null +++ b/backend/python/longcat-video/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +backend_dir=$(dirname "$0") +if [ -d "${backend_dir}/common" ]; then + source "${backend_dir}/common/libbackend.sh" +else + source "${backend_dir}/../common/libbackend.sh" +fi + +runUnittests diff --git a/core/backend/video.go b/core/backend/video.go index e016d1a22..2702fefe9 100644 --- a/core/backend/video.go +++ b/core/backend/video.go @@ -1,21 +1,37 @@ package backend import ( + "maps" "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/trace" - "github.com/mudler/LocalAI/pkg/grpc/proto" model "github.com/mudler/LocalAI/pkg/model" ) -func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, endImage, dst string, numFrames, fps, seed int32, cfgScale float32, step int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { +// VideoGenerationOptions is the backend-neutral request passed to video generators. +// Media fields contain staged local paths by the time they reach this layer. +type VideoGenerationOptions struct { + Height int32 + Width int32 + Prompt string + NegativePrompt string + StartImage string + EndImage string + Audio string + Destination string + NumFrames int32 + FPS int32 + Seed int32 + CFGScale float32 + Step int32 + Params map[string]string +} +func VideoGeneration(options VideoGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { opts := ModelOptions(modelConfig, appConfig) - inferenceModel, err := loader.Load( - opts..., - ) + inferenceModel, err := loader.Load(opts...) if err != nil { recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err @@ -25,19 +41,22 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en _, err := inferenceModel.GenerateVideo( appConfig.Context, &proto.GenerateVideoRequest{ - Height: height, - Width: width, - Prompt: prompt, - NegativePrompt: negativePrompt, - StartImage: startImage, - EndImage: endImage, - NumFrames: numFrames, - Fps: fps, - Seed: seed, - CfgScale: cfgScale, - Step: step, - Dst: dst, - }) + Height: options.Height, + Width: options.Width, + Prompt: options.Prompt, + NegativePrompt: options.NegativePrompt, + StartImage: options.StartImage, + EndImage: options.EndImage, + Audio: options.Audio, + NumFrames: options.NumFrames, + Fps: options.FPS, + Seed: options.Seed, + CfgScale: options.CFGScale, + Step: options.Step, + Dst: options.Destination, + Params: maps.Clone(options.Params), + }, + ) return err } @@ -45,15 +64,18 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) traceData := map[string]any{ - "prompt": prompt, - "negative_prompt": negativePrompt, - "height": height, - "width": width, - "num_frames": numFrames, - "fps": fps, - "seed": seed, - "cfg_scale": cfgScale, - "step": step, + "prompt": options.Prompt, + "negative_prompt": options.NegativePrompt, + "height": options.Height, + "width": options.Width, + "num_frames": options.NumFrames, + "fps": options.FPS, + "seed": options.Seed, + "cfg_scale": options.CFGScale, + "step": options.Step, + "has_start_image": options.StartImage != "", + "has_end_image": options.EndImage != "", + "has_audio": options.Audio != "", } startTime := time.Now() @@ -73,7 +95,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en Type: trace.BackendTraceVideoGeneration, ModelName: modelConfig.Name, Backend: modelConfig.Backend, - Summary: trace.TruncateString(prompt, 200), + Summary: trace.TruncateString(options.Prompt, 200), Error: errStr, Data: traceData, }) diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index d54463a8e..e98d406c9 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -120,7 +120,7 @@ var UsecaseInfoMap = map[string]UsecaseInfo{ UsecaseVideo: { Flag: FLAG_VIDEO, GRPCMethod: MethodGenerateVideo, - Description: "Video generation via the GenerateVideo RPC.", + Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.", }, UsecaseTranscript: { Flag: FLAG_TRANSCRIPT, @@ -303,6 +303,14 @@ var BackendCapabilities = map[string]BackendCapability{ DefaultUsecases: []string{UsecaseImage}, Description: "HuggingFace diffusers — Stable Diffusion, Flux, video generation", }, + "longcat-video": { + GRPCMethods: []GRPCMethod{MethodGenerateVideo}, + PossibleUsecases: []string{UsecaseVideo}, + DefaultUsecases: []string{UsecaseVideo}, + AcceptsImages: true, + AcceptsAudios: true, + Description: "LongCat-Video — text, image, and audio-conditioned avatar video generation on NVIDIA CUDA", + }, "stablediffusion": { GRPCMethods: []GRPCMethod{MethodGenerateImage}, PossibleUsecases: []string{UsecaseImage}, diff --git a/core/config/meta/constants.go b/core/config/meta/constants.go index 7fed6ba75..19eebcb50 100644 --- a/core/config/meta/constants.go +++ b/core/config/meta/constants.go @@ -79,6 +79,14 @@ var UsecaseOptions = []FieldOption{ {Value: "video", Label: "Video"}, } +// ModalityOptions enumerates the values accepted by known modality fields. +var ModalityOptions = []FieldOption{ + {Value: "text", Label: "Text"}, + {Value: "image", Label: "Image"}, + {Value: "audio", Label: "Audio"}, + {Value: "video", Label: "Video"}, +} + var DiffusersSchedulerOptions = []FieldOption{ {Value: "ddim", Label: "DDIM"}, {Value: "ddpm", Label: "DDPM"}, diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 4fa555d65..83785e695 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -66,6 +66,22 @@ func DefaultRegistry() map[string]FieldMetaOverride { Options: UsecaseOptions, Order: 6, }, + "known_input_modalities": { + Section: "general", + Label: "Known Input Modalities", + Description: "Explicit input types this model accepts when use cases alone are not specific enough", + Component: "string-list", + Options: ModalityOptions, + Order: 7, + }, + "known_output_modalities": { + Section: "general", + Label: "Known Output Modalities", + Description: "Explicit output types this model produces when use cases alone are not specific enough", + Component: "string-list", + Options: ModalityOptions, + Order: 8, + }, // --- LLM --- "context_size": { diff --git a/core/config/model_capabilities.go b/core/config/model_capabilities.go index 51b244675..79a117b4a 100644 --- a/core/config/model_capabilities.go +++ b/core/config/model_capabilities.go @@ -1,5 +1,7 @@ package config +import "slices" + // This file is the single source of truth for deriving a model's user-facing // capabilities and input/output modalities from its ModelConfig. Both the // OpenAI-compatible /v1/models/capabilities endpoint and the Ollama-compatible @@ -7,14 +9,47 @@ package config // across clients. Keep the detection heuristics here rather than duplicating // them per endpoint. +// Canonical model modality values used by config declarations and discovery APIs. +const ( + ModalityText = "text" + ModalityImage = "image" + ModalityAudio = "audio" + ModalityVideo = "video" +) + +var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo} + +func declaredModalities(modalities []string) map[string]bool { + declared := make(map[string]bool, len(modalities)) + for _, modality := range modalities { + if slices.Contains(modalityOrder, modality) { + declared[modality] = true + } + } + return declared +} + +func orderedModalities(modalities map[string]bool) []string { + result := make([]string, 0, len(modalityOrder)) + for _, modality := range modalityOrder { + if modalities[modality] { + result = append(result, modality) + } + } + return result +} + // VisionSupported reports whether the model can accept image inputs. // // We deliberately avoid HasUsecases(FLAG_VISION): GuessUsecases has no // FLAG_VISION branch and reports true for any chat model, so it would paint // vision onto text-only models. Instead we look for explicit signals: the -// declared KnownUsecases bit, a multimodal projector, or a template/backend -// multimodal marker. +// declared input modality or KnownUsecases bit, a multimodal projector, or a +// template/backend multimodal marker. func (c *ModelConfig) VisionSupported() bool { + if slices.Contains(c.KnownInputModalities, ModalityImage) { + return true + } if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION { return true } @@ -68,20 +103,21 @@ func (c *ModelConfig) ThinkingSupported() bool { } // AudioInputSupported reports whether a chat/generation model accepts audio as -// input (e.g. vLLM omni models). The signal is the vLLM per-prompt audio limit; -// there is no FLAG_* for "chat model that hears audio", which is exactly why a -// plain usecase list can't express it. Transcription models are handled -// separately in InputModalities via FLAG_TRANSCRIPT. +// input. Model configs can declare this directly; vLLM-family configs can also +// signal it through the per-prompt audio limit. Transcription models are +// handled separately in InputModalities via FLAG_TRANSCRIPT. func (c *ModelConfig) AudioInputSupported() bool { - return c.LimitMMPerPrompt.LimitAudioPerPrompt > 0 + return slices.Contains(c.KnownInputModalities, ModalityAudio) || + c.LimitMMPerPrompt.LimitAudioPerPrompt > 0 } // VideoInputSupported reports whether a chat/generation model accepts video as -// input. The signal is the vLLM per-prompt video limit. Note this is distinct -// from FLAG_VIDEO, which denotes video *generation* (diffusers) — an output -// modality, not an input one. +// input. Model configs can declare this directly; vLLM-family configs can also +// signal it through the per-prompt video limit. This is distinct from +// FLAG_VIDEO, which denotes video generation — an output modality. func (c *ModelConfig) VideoInputSupported() bool { - return c.LimitMMPerPrompt.LimitVideoPerPrompt > 0 + return slices.Contains(c.KnownInputModalities, ModalityVideo) || + c.LimitMMPerPrompt.LimitVideoPerPrompt > 0 } // Capabilities returns the ordered list of capability strings the model @@ -135,6 +171,7 @@ func (c *ModelConfig) Capabilities() []string { // attachment router consults to decide whether an image/audio/video file can be // handed to the active model directly. func (c *ModelConfig) InputModalities() []string { + modalities := declaredModalities(c.KnownInputModalities) imageGen := c.HasUsecases(FLAG_IMAGE) videoGen := c.HasUsecases(FLAG_VIDEO) chatish := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) @@ -154,25 +191,17 @@ func (c *ModelConfig) InputModalities() []string { videoIn := c.VideoInputSupported() - var mods []string - if textIn { - mods = append(mods, "text") - } - if imageIn { - mods = append(mods, "image") - } - if audioIn { - mods = append(mods, "audio") - } - if videoIn { - mods = append(mods, "video") - } - return mods + modalities[ModalityText] = modalities[ModalityText] || textIn + modalities[ModalityImage] = modalities[ModalityImage] || imageIn + modalities[ModalityAudio] = modalities[ModalityAudio] || audioIn + modalities[ModalityVideo] = modalities[ModalityVideo] || videoIn + return orderedModalities(modalities) } // OutputModalities returns the set of modalities (text, image, audio, video) // the model produces, ordered text→image→audio→video. func (c *ModelConfig) OutputModalities() []string { + modalities := declaredModalities(c.KnownOutputModalities) textOut := c.HasUsecases(FLAG_CHAT) || c.HasUsecases(FLAG_COMPLETION) || c.HasUsecases(FLAG_EDIT) || c.HasUsecases(FLAG_TRANSCRIPT) imageOut := c.HasUsecases(FLAG_IMAGE) @@ -180,18 +209,9 @@ func (c *ModelConfig) OutputModalities() []string { c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) videoOut := c.HasUsecases(FLAG_VIDEO) - var mods []string - if textOut { - mods = append(mods, "text") - } - if imageOut { - mods = append(mods, "image") - } - if audioOut { - mods = append(mods, "audio") - } - if videoOut { - mods = append(mods, "video") - } - return mods + modalities[ModalityText] = modalities[ModalityText] || textOut + modalities[ModalityImage] = modalities[ModalityImage] || imageOut + modalities[ModalityAudio] = modalities[ModalityAudio] || audioOut + modalities[ModalityVideo] = modalities[ModalityVideo] || videoOut + return orderedModalities(modalities) } diff --git a/core/config/model_capabilities_test.go b/core/config/model_capabilities_test.go index 8aab180b2..545bad50b 100644 --- a/core/config/model_capabilities_test.go +++ b/core/config/model_capabilities_test.go @@ -21,6 +21,14 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.VisionSupported()).To(BeTrue()) }) + It("is true when image input is declared explicitly", func() { + cfg := &ModelConfig{ + KnownUsecases: usecaseBits(FLAG_CHAT), + KnownInputModalities: []string{ModalityText, ModalityImage}, + } + Expect(cfg.VisionSupported()).To(BeTrue()) + }) + It("is true when an mmproj projector is set", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "llama.cpp"} cfg.MMProj = "mmproj.gguf" // promoted field from the embedded options struct @@ -37,6 +45,14 @@ var _ = Describe("Model capabilities derivation", func() { }) Describe("AudioInputSupported / VideoInputSupported", func() { + It("honors explicit model modality declarations", func() { + cfg := &ModelConfig{ + KnownInputModalities: []string{ModalityAudio, ModalityVideo}, + } + Expect(cfg.AudioInputSupported()).To(BeTrue()) + Expect(cfg.VideoInputSupported()).To(BeTrue()) + }) + It("detects vLLM omni audio input via limit_mm_per_prompt", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_CHAT), Backend: "vllm"} cfg.LimitMMPerPrompt.LimitAudioPerPrompt = 1 @@ -93,6 +109,17 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.OutputModalities()).To(Equal([]string{"image"})) }) + It("conditioned video uses declared modalities without backend-specific inference", func() { + cfg := &ModelConfig{ + KnownUsecases: usecaseBits(FLAG_VIDEO), + KnownInputModalities: []string{ModalityAudio, ModalityImage, ModalityText, ModalityAudio, "unknown"}, + KnownOutputModalities: []string{ModalityVideo}, + } + Expect(cfg.Capabilities()).To(Equal([]string{UsecaseVideo})) + Expect(cfg.InputModalities()).To(Equal([]string{ModalityText, ModalityImage, ModalityAudio})) + Expect(cfg.OutputModalities()).To(Equal([]string{ModalityVideo})) + }) + It("a TTS model reads text and writes audio", func() { cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "piper"} Expect(cfg.Capabilities()).To(ContainElement(UsecaseTTS)) diff --git a/core/config/model_config.go b/core/config/model_config.go index 0038f4f8d..5cdcbdd7c 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -52,7 +52,11 @@ type ModelConfig struct { TemplateConfig TemplateConfig `yaml:"template,omitempty" json:"template,omitempty"` KnownUsecaseStrings []string `yaml:"known_usecases,omitempty" json:"known_usecases,omitempty"` KnownUsecases *ModelConfigUsecase `yaml:"-" json:"-"` - Pipeline Pipeline `yaml:"pipeline,omitempty" json:"pipeline,omitempty"` + // KnownInputModalities and KnownOutputModalities describe model-specific I/O + // that usecases alone cannot express, such as image- or audio-conditioned video. + KnownInputModalities []string `yaml:"known_input_modalities,omitempty" json:"known_input_modalities,omitempty"` + KnownOutputModalities []string `yaml:"known_output_modalities,omitempty" json:"known_output_modalities,omitempty"` + Pipeline Pipeline `yaml:"pipeline,omitempty" json:"pipeline,omitempty"` PromptStrings, InputStrings []string `yaml:"-" json:"-"` InputToken [][]int `yaml:"-" json:"-"` diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 3db8d88f7..3fc40e02c 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -33,7 +33,7 @@ var ErrAmbiguousImport = errors.New("importer: ambiguous — specify preferences // pipeline_tag values. type AmbiguousImportError struct { // Modality is the importer modality key ("text", "asr", "tts", "image", - // "embeddings", "reranker", "detection"). Pre-mapped from the HF + // "video", "embeddings", "reranker", "detection"). Pre-mapped from the HF // pipeline_tag so the UI doesn't have to. Modality string // Candidates is the list of backend names whose Modality() matches — a @@ -144,6 +144,9 @@ var defaultImporters = []Importer{ // Image/Video (Batch 3) &StableDiffusionGGMLImporter{}, &ACEStepImporter{}, + // LongCat repositories carry generic Diffusers metadata, so this exact + // owner/repo matcher must run before DiffuserImporter. + &LongCatVideoImporter{}, // Text LLM (Batch 4) — VLLMOmniImporter must stay ahead of // VLLMImporter so Qwen Omni repos (which also carry tokenizer // files) route to vllm-omni rather than plain vllm. @@ -204,7 +207,7 @@ type Importer interface { // /backends/known to populate the import form dropdown. Name() string // Modality is the backend's primary modality ("text", "asr", "tts", - // "image", "embeddings", "reranker", "detection", "vad"). Used for + // "image", "video", "embeddings", "reranker", "detection", "vad"). Used for // grouping in the UI. Modality() string // AutoDetects is true when Match() can fire without an explicit diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go index d29f59182..7e34b7b3e 100644 --- a/core/gallery/importers/importers_test.go +++ b/core/gallery/importers/importers_test.go @@ -15,6 +15,16 @@ import ( var _ = Describe("DiscoverModelConfig", func() { Context("With only a repository URI", func() { + It("should discover LongCat Avatar before the generic Diffusers importer", func() { + uri := "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5" + + modelConfig, err := importers.DiscoverModelConfig(uri, json.RawMessage(`{}`)) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases:\n - video")) + }) + It("should discover and import using LlamaCPPImporter", func() { uri := "https://huggingface.co/mudler/LocalAI-functioncall-qwen2.5-7b-v0.5-Q4_K_M-GGUF" preferences := json.RawMessage(`{}`) @@ -242,7 +252,7 @@ var _ = Describe("DiscoverModelConfig", func() { for _, imp := range registry { names = append(names, imp.Name()) } - Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers")) + Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers", "longcat-video")) }) It("LlamaCPPImporter exposes name/modality/autodetect", func() { diff --git a/core/gallery/importers/longcat-video.go b/core/gallery/importers/longcat-video.go new file mode 100644 index 000000000..f73ae9136 --- /dev/null +++ b/core/gallery/importers/longcat-video.go @@ -0,0 +1,127 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "gopkg.in/yaml.v3" +) + +const ( + longCatOwner = "meituan-longcat" + longCatBaseRepo = "LongCat-Video" + longCatAvatarRepo = "LongCat-Video-Avatar-1.5" +) + +var _ Importer = &LongCatVideoImporter{} + +// LongCatVideoImporter is deliberately owner/repository-specific. LongCat +// checkpoints also contain generic Diffusers metadata, so broad file-based +// matching would let this backend claim unrelated video pipelines. +type LongCatVideoImporter struct{} + +func (i *LongCatVideoImporter) Name() string { return "longcat-video" } +func (i *LongCatVideoImporter) Modality() string { return "video" } +func (i *LongCatVideoImporter) AutoDetects() bool { return true } + +func isLongCatRepo(owner, repo string) bool { + repo = strings.Split(strings.Trim(repo, "/"), "/")[0] + return strings.EqualFold(owner, longCatOwner) && + (strings.EqualFold(repo, longCatBaseRepo) || strings.EqualFold(repo, longCatAvatarRepo)) +} + +func longCatModelID(details Details) (string, bool) { + if details.HuggingFace != nil && details.HuggingFace.ModelID != "" { + parts := strings.SplitN(details.HuggingFace.ModelID, "/", 2) + if len(parts) == 2 && isLongCatRepo(parts[0], parts[1]) { + repo := strings.Split(strings.Trim(parts[1], "/"), "/")[0] + return parts[0] + "/" + repo, true + } + } + if owner, repo, ok := HFOwnerRepoFromURI(details.URI); ok && isLongCatRepo(owner, repo) { + repo = strings.Split(strings.Trim(repo, "/"), "/")[0] + return owner + "/" + repo, true + } + return LocalModelPath(details.URI), false +} + +func (i *LongCatVideoImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return false + } + } + if backend, ok := preferencesMap["backend"].(string); ok { + return backend == i.Name() + } + + _, matched := longCatModelID(details) + return matched +} + +func (i *LongCatVideoImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return gallery.ModelConfig{}, err + } + } + + model, canonical := longCatModelID(details) + name, _ := preferencesMap["name"].(string) + if name == "" { + if canonical { + name = strings.ToLower(filepath.Base(model)) + } else { + name = filepath.Base(strings.TrimSuffix(model, "/")) + } + } + + description, _ := preferencesMap["description"].(string) + if description == "" { + description = "Imported from " + details.URI + } + + options := []string{"attention_backend:sdpa"} + inputModalities := []string{config.ModalityText, config.ModalityImage} + if strings.EqualFold(filepath.Base(model), longCatAvatarRepo) { + options = append(options, "use_distill:true") + inputModalities = append(inputModalities, config.ModalityAudio) + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + Backend: i.Name(), + KnownUsecaseStrings: []string{config.UsecaseVideo}, + KnownInputModalities: inputModalities, + KnownOutputModalities: []string{config.ModalityVideo}, + Options: options, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{Model: model}, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/longcat-video_test.go b/core/gallery/importers/longcat-video_test.go new file mode 100644 index 000000000..6abced791 --- /dev/null +++ b/core/gallery/importers/longcat-video_test.go @@ -0,0 +1,116 @@ +package importers_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +func decodeLongCatModelConfig(data string) config.ModelConfig { + GinkgoHelper() + modelConfig := config.ModelConfig{} + Expect(yaml.Unmarshal([]byte(data), &modelConfig)).To(Succeed()) + return modelConfig +} + +var _ = Describe("LongCatVideoImporter", func() { + var importer *importers.LongCatVideoImporter + + BeforeEach(func() { + importer = &importers.LongCatVideoImporter{} + }) + + It("exposes video importer metadata", func() { + Expect(importer.Name()).To(Equal("longcat-video")) + Expect(importer.Modality()).To(Equal("video")) + Expect(importer.AutoDetects()).To(BeTrue()) + }) + + Describe("Match", func() { + It("matches both official repositories", func() { + for _, modelID := range []string{ + "meituan-longcat/LongCat-Video", + "meituan-longcat/LongCat-Video-Avatar-1.5", + } { + details := importers.Details{ + URI: "https://huggingface.co/" + modelID, + HuggingFace: &hfapi.ModelDetails{ + ModelID: modelID, + Author: "meituan-longcat", + }, + } + Expect(importer.Match(details)).To(BeTrue(), modelID) + } + }) + + It("matches official hf URI forms without metadata", func() { + Expect(importer.Match(importers.Details{ + URI: "https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5/tree/main/", + })).To(BeTrue()) + }) + + It("does not claim the same repository name under another owner", func() { + Expect(importer.Match(importers.Details{ + URI: "https://huggingface.co/other-org/LongCat-Video", + })).To(BeFalse()) + }) + + It("honors an explicit backend preference", func() { + Expect(importer.Match(importers.Details{ + URI: "/models/LongCat-Video", + Preferences: json.RawMessage(`{"backend":"longcat-video"}`), + })).To(BeTrue()) + Expect(importer.Match(importers.Details{ + URI: "hf://meituan-longcat/LongCat-Video", + Preferences: json.RawMessage(`{"backend":"diffusers"}`), + })).To(BeFalse()) + }) + }) + + Describe("Import", func() { + It("emits a base-model video configuration", func() { + modelConfig, err := importer.Import(importers.Details{ + URI: "https://huggingface.co/meituan-longcat/LongCat-Video", + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: longcat-video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- video")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("attention_backend:sdpa")) + Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("use_distill:true")) + + cfg := decodeLongCatModelConfig(modelConfig.ConfigFile) + Expect(cfg.KnownInputModalities).To(Equal([]string{ + config.ModalityText, + config.ModalityImage, + })) + Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo})) + }) + + It("enables the distilled path for Avatar 1.5", func() { + modelConfig, err := importer.Import(importers.Details{ + URI: "hf://meituan-longcat/LongCat-Video-Avatar-1.5", + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(modelConfig.Name).To(Equal("longcat-video-avatar-1.5")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: meituan-longcat/LongCat-Video-Avatar-1.5")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("use_distill:true")) + + cfg := decodeLongCatModelConfig(modelConfig.ConfigFile) + Expect(cfg.KnownInputModalities).To(Equal([]string{ + config.ModalityText, + config.ModalityImage, + config.ModalityAudio, + })) + Expect(cfg.KnownOutputModalities).To(Equal([]string{config.ModalityVideo})) + }) + }) +}) diff --git a/core/http/app_test.go b/core/http/app_test.go index 5917b034a..fa6423292 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -983,7 +983,22 @@ chat_template_kwargs: req.Header.Set("Authorization", bearerKey) resp, err = http.DefaultClient.Do(req) Expect(err).ToNot(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(200)) + if resp.StatusCode == http.StatusBadRequest { + // The worker can finish between the status read and cancellation request. + resp, err = http.Get("http://127.0.0.1:9090/api/agent/jobs/" + jobID) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + body, _ = io.ReadAll(resp.Body) + err = json.Unmarshal(body, &job) + Expect(err).ToNot(HaveOccurred()) + Expect(job.Status).To(Or( + Equal(schema.JobStatusCompleted), + Equal(schema.JobStatusFailed), + Equal(schema.JobStatusCancelled), + )) + } else { + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + } } }) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 405921e5e..13994d285 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -71,8 +71,9 @@ var instructionDefs = []instructionDef{ }, { Name: "video", - Description: "Video generation from text prompts", + Description: "Video generation from text prompts with optional image or audio conditioning", Tags: []string{"video"}, + Intro: "POST /video accepts start_image, end_image, and audio as public URL, base64, or data URI. Backend-specific tuning is passed as string values in params.", }, { Name: "face-recognition", diff --git a/core/http/endpoints/localai/video.go b/core/http/endpoints/localai/video.go index 65c140dc2..9e6a33e10 100644 --- a/core/http/endpoints/localai/video.go +++ b/core/http/endpoints/localai/video.go @@ -1,11 +1,12 @@ package localai import ( - "bufio" + "context" "encoding/base64" "encoding/json" "fmt" "io" + "net/http" "net/url" "os" "path/filepath" @@ -28,32 +29,105 @@ import ( "github.com/mudler/LocalAI/pkg/utils" ) -// Downloading user-supplied media URLs legitimately follows redirects (CDNs); -// WithFollowRedirects still strips any credential header on a cross-host hop. -var videoDownloadClient = httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) +const maxVideoInputBytes = 128 << 20 -func downloadFile(url string) (string, error) { - if err := utils.ValidateExternalURL(url); err != nil { - return "", fmt.Errorf("URL validation failed: %w", err) +func newVideoDownloadClient() *http.Client { + client := httpclient.NewWithTimeout(30*time.Second, httpclient.WithFollowRedirects()) + checkRedirect := client.CheckRedirect + // Media CDNs commonly redirect, so validate every hop rather than trusting + // only the URL supplied by the caller. Keep the shared redirect policy too; + // it bounds the chain and strips credentials on cross-origin hops. + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if err := utils.ValidateExternalURL(req.URL.String()); err != nil { + return fmt.Errorf("redirect URL validation failed: %w", err) + } + return checkRedirect(req, via) + } + return client +} + +var videoDownloadClient = newVideoDownloadClient() + +func openVideoMedia(ctx context.Context, ref string) (io.ReadCloser, int64, error) { + if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { + if err := utils.ValidateExternalURL(ref); err != nil { + return nil, 0, fmt.Errorf("URL validation failed: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil) + if err != nil { + return nil, 0, fmt.Errorf("creating download request: %w", err) + } + resp, err := videoDownloadClient.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("downloading media: %w", err) + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + _ = resp.Body.Close() + return nil, 0, fmt.Errorf("media URL returned HTTP %d", resp.StatusCode) + } + return resp.Body, resp.ContentLength, nil } - // Get the data - resp, err := videoDownloadClient.Get(url) + encoded := ref + if strings.HasPrefix(ref, "data:") { + comma := strings.IndexByte(ref, ',') + if comma < 0 || !strings.Contains(strings.ToLower(ref[:comma]), ";base64") { + return nil, 0, fmt.Errorf("data URI must contain a base64 payload") + } + encoded = ref[comma+1:] + } + if encoded == "" { + return nil, 0, fmt.Errorf("media payload is empty") + } + + decodedSize := int64(base64.StdEncoding.DecodedLen(len(encoded))) + return io.NopCloser(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))), decodedSize, nil +} + +func stageVideoMedia(ctx context.Context, directory, ref string) (string, error) { + return stageVideoMediaWithLimit(ctx, directory, ref, maxVideoInputBytes) +} + +func stageVideoMediaWithLimit(ctx context.Context, directory, ref string, maxBytes int64) (string, error) { + if ref == "" { + return "", nil + } + + source, declaredSize, err := openVideoMedia(ctx, ref) if err != nil { return "", err } - defer resp.Body.Close() - - // Create the file - out, err := os.CreateTemp("", "video") - if err != nil { - return "", err + defer func() { _ = source.Close() }() + if declaredSize > maxBytes { + return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes) } - defer out.Close() - // Write the body to file - _, err = io.Copy(out, resp.Body) - return out.Name(), err + output, err := os.CreateTemp(directory, "video-input-*") + if err != nil { + return "", fmt.Errorf("creating staged media file: %w", err) + } + outputPath := output.Name() + keep := false + defer func() { + _ = output.Close() + if !keep { + _ = os.Remove(outputPath) + } + }() + + written, err := io.Copy(output, io.LimitReader(source, maxBytes+1)) + if err != nil { + return "", fmt.Errorf("decoding media: %w", err) + } + if written > maxBytes { + return "", fmt.Errorf("media exceeds the %d-byte limit", maxBytes) + } + if err := output.Close(); err != nil { + return "", fmt.Errorf("closing staged media file: %w", err) + } + keep = true + return outputPath, nil } // @@ -72,7 +146,7 @@ func downloadFile(url string) (string, error) { * */ // VideoEndpoint -// @Summary Creates a video given a prompt. +// @Summary Creates a video from a prompt and optional image or audio conditioning. // @Tags video // @Param request body schema.VideoRequest true "query params" // @Success 200 {object} schema.OpenAIResponse "Response" @@ -91,63 +165,39 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi return echo.ErrBadRequest } - // Stage a base64- or URL-provided image into a temp file so the - // backend can read it as a path. Used for both start_image and - // (optional) end_image. Returns the temp file path, or "" if the - // input is empty. Caller is responsible for the defer-cleanup. - stageImage := func(ref string) (string, error) { - if ref == "" { - return "", nil - } - var fileData []byte - var err error - if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { - out, derr := downloadFile(ref) - if derr != nil { - return "", fmt.Errorf("failed downloading file: %w", derr) - } - defer os.RemoveAll(out) - fileData, err = os.ReadFile(out) - if err != nil { - return "", fmt.Errorf("failed reading file: %w", err) - } - } else { - fileData, err = base64.StdEncoding.DecodeString(ref) - if err != nil { - return "", err - } - } - outputFile, err := os.CreateTemp(appConfig.GeneratedContentDir, "b64") + stageInput := func(name, ref string) (string, error) { + path, err := stageVideoMedia(c.Request().Context(), appConfig.GeneratedContentDir, ref) if err != nil { - return "", err + return "", echo.NewHTTPError( + http.StatusBadRequest, + fmt.Sprintf("invalid %s: %v", name, err), + ) } - writer := bufio.NewWriter(outputFile) - if _, err := writer.Write(fileData); err != nil { - outputFile.Close() - return "", err - } - if err := writer.Flush(); err != nil { - outputFile.Close() - return "", err - } - outputFile.Close() - return outputFile.Name(), nil + return path, nil } - src, err := stageImage(input.StartImage) + src, err := stageInput("start_image", input.StartImage) if err != nil { return err } if src != "" { - defer os.RemoveAll(src) + defer func() { _ = os.Remove(src) }() } - endSrc, err := stageImage(input.EndImage) + endSrc, err := stageInput("end_image", input.EndImage) if err != nil { return err } if endSrc != "" { - defer os.RemoveAll(endSrc) + defer func() { _ = os.Remove(endSrc) }() + } + + audioSrc, err := stageInput("audio", input.Audio) + if err != nil { + return err + } + if audioSrc != "" { + defer func() { _ = os.Remove(audioSrc) }() } xlog.Debug("Parameter Config", "config", config) @@ -174,13 +224,19 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi tempDir := "" if !b64JSON { tempDir = filepath.Join(appConfig.GeneratedContentDir, "videos") + if err := os.MkdirAll(tempDir, 0o750); err != nil { + return err + } } // Create a temporary file outputFile, err := os.CreateTemp(tempDir, "b64") if err != nil { return err } - outputFile.Close() + if err := outputFile.Close(); err != nil { + _ = os.Remove(outputFile.Name()) + return err + } // TODO: use mime type to determine the extension output := outputFile.Name() + ".mp4" @@ -188,8 +244,15 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi // Rename the temporary file err = os.Rename(outputFile.Name(), output) if err != nil { + _ = os.Remove(outputFile.Name()) return err } + preserveOutput := false + defer func() { + if !preserveOutput { + _ = os.Remove(output) + } + }() baseURL := middleware.BaseURL(c) @@ -204,33 +267,36 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi "negative_prompt", input.NegativePrompt) fn, err := backend.VideoGeneration( - height, - width, - input.Prompt, - input.NegativePrompt, - src, - endSrc, - output, - input.NumFrames, - input.FPS, - input.Seed, - input.CFGScale, - input.Step, + backend.VideoGenerationOptions{ + Height: height, + Width: width, + Prompt: input.Prompt, + NegativePrompt: input.NegativePrompt, + StartImage: src, + EndImage: endSrc, + Audio: audioSrc, + Destination: output, + NumFrames: input.NumFrames, + FPS: input.FPS, + Seed: input.Seed, + CFGScale: input.CFGScale, + Step: input.Step, + Params: input.Params, + }, ml, *config, appConfig, ) if err != nil { - return err + return mapBackendError(err) } if err := fn(); err != nil { - return err + return mapBackendError(err) } item := &schema.Item{} if b64JSON { - defer os.RemoveAll(output) data, err := os.ReadFile(output) if err != nil { return err @@ -242,6 +308,7 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi if err != nil { return err } + preserveOutput = true } id := uuid.New().String() diff --git a/core/http/endpoints/localai/video_internal_test.go b/core/http/endpoints/localai/video_internal_test.go new file mode 100644 index 000000000..383ae5fe4 --- /dev/null +++ b/core/http/endpoints/localai/video_internal_test.go @@ -0,0 +1,68 @@ +package localai + +import ( + "context" + "encoding/base64" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("video media staging", func() { + It("stages raw base64 into a private temporary file", func() { + directory := GinkgoT().TempDir() + content := []byte("avatar audio") + + path, err := stageVideoMedia(context.Background(), directory, base64.StdEncoding.EncodeToString(content)) + + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.Remove, path) + Expect(filepath.Dir(path)).To(Equal(directory)) + Expect(os.ReadFile(path)).To(Equal(content)) + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) + + It("accepts browser data URIs with codec parameters", func() { + content := []byte("recorded speech") + ref := "data:audio/webm;codecs=opus;base64," + base64.StdEncoding.EncodeToString(content) + + path, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), ref) + + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.Remove, path) + Expect(os.ReadFile(path)).To(Equal(content)) + }) + + It("rejects malformed base64 and removes the partial file", func() { + directory := GinkgoT().TempDir() + + _, err := stageVideoMedia(context.Background(), directory, "not%%%base64") + + Expect(err).To(MatchError(ContainSubstring("decoding media"))) + entries, readErr := os.ReadDir(directory) + Expect(readErr).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("enforces the configured streaming limit", func() { + directory := GinkgoT().TempDir() + encoded := base64.StdEncoding.EncodeToString([]byte("four")) + + _, err := stageVideoMediaWithLimit(context.Background(), directory, encoded, 3) + + Expect(err).To(MatchError(ContainSubstring("3-byte limit"))) + entries, readErr := os.ReadDir(directory) + Expect(readErr).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + }) + + It("rejects non-base64 data URIs", func() { + _, err := stageVideoMedia(context.Background(), GinkgoT().TempDir(), "data:audio/wav,plain") + + Expect(err).To(MatchError(ContainSubstring("base64 payload"))) + }) +}) diff --git a/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js b/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js index 5532123a7..49881d367 100644 --- a/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js +++ b/core/http/react-ui/e2e/import-form-ux-batch-e.spec.js @@ -23,6 +23,7 @@ const MOCK_BACKENDS = [ { name: 'kokoro', modality: 'tts', auto_detect: true, installed: true }, { name: 'whisper', modality: 'asr', auto_detect: true, installed: true }, { name: 'diffusers', modality: 'image', auto_detect: true, installed: false }, + { name: 'longcat-video', modality: 'video', auto_detect: true, installed: false }, { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true }, { name: 'rerankers', modality: 'reranker', auto_detect: true, installed: true }, { name: 'rfdetr', modality: 'detection', auto_detect: true, installed: true }, @@ -78,7 +79,7 @@ test.describe('Import form UX — Batch E (modality chip row)', () => { await page.locator('[data-testid="simple-options-toggle"]').click() await expect(chips(page)).toBeVisible() // Full set of chips renders. - for (const key of ['', 'text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad']) { + for (const key of ['', 'text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad']) { await expect(chip(page, key)).toBeVisible() } // "Any" is active by default. diff --git a/core/http/react-ui/e2e/media-history.spec.js b/core/http/react-ui/e2e/media-history.spec.js index 754393973..7575388f7 100644 --- a/core/http/react-ui/e2e/media-history.spec.js +++ b/core/http/react-ui/e2e/media-history.spec.js @@ -21,9 +21,10 @@ function mockImageGeneration(page, images) { }) } -function mockVideoGeneration(page, videos) { +function mockVideoGeneration(page, videos, onRequest) { return page.route('**/video', (route) => { if (route.request().method() !== 'POST') return route.continue() + onRequest?.(route.request().postDataJSON()) route.fulfill({ contentType: 'application/json', body: JSON.stringify({ @@ -254,4 +255,22 @@ test.describe('Media History - Video Generation', () => { await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 10_000 }) await expect(page.getByTestId('media-history-item')).toContainText('a running cat') }) + + test('Avatar audio is forwarded to the video API', async ({ page }) => { + let requestBody + await mockVideoGeneration(page, [{ url: '/generated-videos/avatar.mp4' }], (body) => { requestBody = body }) + + await page.goto('/app/video') + await expect(page.getByRole('button', { name: 'test-video-model' })).toBeVisible({ timeout: 10_000 }) + await page.getByRole('button', { name: 'Reference media' }).click() + await page.getByLabel('Avatar audio', { exact: true }).setInputFiles({ + name: 'speech.wav', + mimeType: 'audio/wav', + buffer: Buffer.from('avatar speech'), + }) + await page.locator('.textarea').first().fill('a presenter speaking to camera') + await page.locator('button[type="submit"]').click() + + await expect.poll(() => requestBody?.audio).toBe(Buffer.from('avatar speech').toString('base64')) + }) }) diff --git a/core/http/react-ui/public/locales/de/importModel.json b/core/http/react-ui/public/locales/de/importModel.json index 67623b955..373cc309b 100644 --- a/core/http/react-ui/public/locales/de/importModel.json +++ b/core/http/react-ui/public/locales/de/importModel.json @@ -72,6 +72,7 @@ "asr": "Spracherkennung", "tts": "Sprachsynthese", "image": "Bild / Video", + "video": "Videogenerierung", "embeddings": "Embeddings", "reranker": "Reranker", "detection": "Objekterkennung", diff --git a/core/http/react-ui/public/locales/de/media.json b/core/http/react-ui/public/locales/de/media.json index deb4d7631..096c74b36 100644 --- a/core/http/react-ui/public/locales/de/media.json +++ b/core/http/react-ui/public/locales/de/media.json @@ -52,7 +52,12 @@ "size": "Größe", "advanced": "Erweiterte Einstellungen", "seed": "Seed", - "seedPlaceholder": "Zufällig" + "seedPlaceholder": "Zufällig", + "frames": "Frames", + "referenceMedia": "Referenzmedien", + "startImage": "Startbild", + "endImage": "Endbild", + "avatarAudio": "Avatar-Audio" }, "actions": { "generate": "Video generieren", diff --git a/core/http/react-ui/public/locales/en/importModel.json b/core/http/react-ui/public/locales/en/importModel.json index 0a67d1613..ad040f3c2 100644 --- a/core/http/react-ui/public/locales/en/importModel.json +++ b/core/http/react-ui/public/locales/en/importModel.json @@ -72,6 +72,7 @@ "asr": "Speech recognition", "tts": "Text-to-speech", "image": "Image / Video", + "video": "Video generation", "embeddings": "Embeddings", "reranker": "Rerankers", "detection": "Object detection", diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 4cff8ffa4..ae644a3fa 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -52,7 +52,12 @@ "size": "Size", "advanced": "Advanced Settings", "seed": "Seed", - "seedPlaceholder": "Random" + "seedPlaceholder": "Random", + "frames": "Frames", + "referenceMedia": "Reference media", + "startImage": "Start image", + "endImage": "End image", + "avatarAudio": "Avatar audio" }, "actions": { "generate": "Generate", diff --git a/core/http/react-ui/public/locales/es/importModel.json b/core/http/react-ui/public/locales/es/importModel.json index 96e447a64..aa29731fb 100644 --- a/core/http/react-ui/public/locales/es/importModel.json +++ b/core/http/react-ui/public/locales/es/importModel.json @@ -72,6 +72,7 @@ "asr": "Reconocimiento de voz", "tts": "Texto a voz", "image": "Imagen / Video", + "video": "Generación de video", "embeddings": "Embeddings", "reranker": "Rerankers", "detection": "Detección de objetos", diff --git a/core/http/react-ui/public/locales/es/media.json b/core/http/react-ui/public/locales/es/media.json index 768b851a5..1a36a3a24 100644 --- a/core/http/react-ui/public/locales/es/media.json +++ b/core/http/react-ui/public/locales/es/media.json @@ -52,7 +52,12 @@ "size": "Tamaño", "advanced": "Configuración avanzada", "seed": "Seed", - "seedPlaceholder": "Aleatorio" + "seedPlaceholder": "Aleatorio", + "frames": "Fotogramas", + "referenceMedia": "Medios de referencia", + "startImage": "Imagen inicial", + "endImage": "Imagen final", + "avatarAudio": "Audio del avatar" }, "actions": { "generate": "Generar video", diff --git a/core/http/react-ui/public/locales/id/importModel.json b/core/http/react-ui/public/locales/id/importModel.json index a23333873..bad709a86 100644 --- a/core/http/react-ui/public/locales/id/importModel.json +++ b/core/http/react-ui/public/locales/id/importModel.json @@ -72,6 +72,7 @@ "asr": "Pengenalan suara", "tts": "Text-to-speech", "image": "Gambar / Video", + "video": "Pembuatan video", "embeddings": "Embedding", "reranker": "Reranker", "detection": "Deteksi object", diff --git a/core/http/react-ui/public/locales/id/media.json b/core/http/react-ui/public/locales/id/media.json index 10350967b..50bbfa65b 100644 --- a/core/http/react-ui/public/locales/id/media.json +++ b/core/http/react-ui/public/locales/id/media.json @@ -52,7 +52,12 @@ "size": "Ukuran", "advanced": "Pengaturan Tingkat Lanjutan", "seed": "Seed", - "seedPlaceholder": "Acak" + "seedPlaceholder": "Acak", + "frames": "Frame", + "referenceMedia": "Media referensi", + "startImage": "Gambar awal", + "endImage": "Gambar akhir", + "avatarAudio": "Audio avatar" }, "actions": { "generate": "Hasilkan", diff --git a/core/http/react-ui/public/locales/it/importModel.json b/core/http/react-ui/public/locales/it/importModel.json index 925f7d0ad..2d5b1391c 100644 --- a/core/http/react-ui/public/locales/it/importModel.json +++ b/core/http/react-ui/public/locales/it/importModel.json @@ -72,6 +72,7 @@ "asr": "Riconoscimento vocale", "tts": "Sintesi vocale", "image": "Immagine / Video", + "video": "Generazione video", "embeddings": "Embedding", "reranker": "Reranker", "detection": "Rilevamento oggetti", diff --git a/core/http/react-ui/public/locales/it/media.json b/core/http/react-ui/public/locales/it/media.json index 06adcf8ee..b41629a1c 100644 --- a/core/http/react-ui/public/locales/it/media.json +++ b/core/http/react-ui/public/locales/it/media.json @@ -52,7 +52,12 @@ "size": "Dimensione", "advanced": "Impostazioni avanzate", "seed": "Seed", - "seedPlaceholder": "Casuale" + "seedPlaceholder": "Casuale", + "frames": "Fotogrammi", + "referenceMedia": "Media di riferimento", + "startImage": "Immagine iniziale", + "endImage": "Immagine finale", + "avatarAudio": "Audio avatar" }, "actions": { "generate": "Genera video", diff --git a/core/http/react-ui/public/locales/ko/importModel.json b/core/http/react-ui/public/locales/ko/importModel.json index 5be2624e4..8ad924e3d 100644 --- a/core/http/react-ui/public/locales/ko/importModel.json +++ b/core/http/react-ui/public/locales/ko/importModel.json @@ -72,6 +72,7 @@ "asr": "음성 인식", "tts": "텍스트 음성 변환", "image": "이미지 / 비디오", + "video": "비디오 생성", "embeddings": "임베딩", "reranker": "리랭커", "detection": "객체 감지", diff --git a/core/http/react-ui/public/locales/ko/media.json b/core/http/react-ui/public/locales/ko/media.json index d17ef41f2..a18abc0eb 100644 --- a/core/http/react-ui/public/locales/ko/media.json +++ b/core/http/react-ui/public/locales/ko/media.json @@ -52,7 +52,12 @@ "size": "크기", "advanced": "고급 설정", "seed": "시드", - "seedPlaceholder": "랜덤" + "seedPlaceholder": "랜덤", + "frames": "프레임", + "referenceMedia": "참조 미디어", + "startImage": "시작 이미지", + "endImage": "종료 이미지", + "avatarAudio": "아바타 오디오" }, "actions": { "generate": "생성", diff --git a/core/http/react-ui/public/locales/zh-CN/importModel.json b/core/http/react-ui/public/locales/zh-CN/importModel.json index 2255a0711..f0800d08d 100644 --- a/core/http/react-ui/public/locales/zh-CN/importModel.json +++ b/core/http/react-ui/public/locales/zh-CN/importModel.json @@ -72,6 +72,7 @@ "asr": "语音识别", "tts": "文字转语音", "image": "图像 / 视频", + "video": "视频生成", "embeddings": "嵌入", "reranker": "重排器", "detection": "对象检测", diff --git a/core/http/react-ui/public/locales/zh-CN/media.json b/core/http/react-ui/public/locales/zh-CN/media.json index 8feb89057..f395cc193 100644 --- a/core/http/react-ui/public/locales/zh-CN/media.json +++ b/core/http/react-ui/public/locales/zh-CN/media.json @@ -52,7 +52,12 @@ "size": "尺寸", "advanced": "高级设置", "seed": "随机种子", - "seedPlaceholder": "随机" + "seedPlaceholder": "随机", + "frames": "帧数", + "referenceMedia": "参考媒体", + "startImage": "起始图像", + "endImage": "结束图像", + "avatarAudio": "虚拟形象音频" }, "actions": { "generate": "生成视频", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index d14aa8f08..82c491f9c 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -2605,6 +2605,24 @@ select.input { user-select: none; } +button.collapsible-header { + width: 100%; + border: 0; + background: transparent; + font-family: inherit; + text-align: left; +} + +button.collapsible-header:hover { + color: var(--color-text-primary); +} + +button.collapsible-header:focus-visible { + border-radius: var(--radius-sm); + outline: 2px solid var(--color-primary); + outline-offset: 2px; +} + .collapsible-header i { transition: transform var(--duration-fast); } diff --git a/core/http/react-ui/src/components/ModalityChips.jsx b/core/http/react-ui/src/components/ModalityChips.jsx index cb6a27b5c..52bfc48cc 100644 --- a/core/http/react-ui/src/components/ModalityChips.jsx +++ b/core/http/react-ui/src/components/ModalityChips.jsx @@ -16,6 +16,7 @@ const CHIPS = [ { key: 'asr', label: 'Speech' }, { key: 'tts', label: 'TTS' }, { key: 'image', label: 'Image' }, + { key: 'video', label: 'Video' }, { key: 'embeddings', label: 'Embeddings' }, { key: 'reranker', label: 'Rerankers' }, { key: 'detection', label: 'Detection' }, diff --git a/core/http/react-ui/src/pages/ImportModel.jsx b/core/http/react-ui/src/pages/ImportModel.jsx index 6ba47ffe8..5adc82303 100644 --- a/core/http/react-ui/src/pages/ImportModel.jsx +++ b/core/http/react-ui/src/pages/ImportModel.jsx @@ -16,7 +16,7 @@ const BACKENDS_FALLBACK_EMPTY = [] // Modality keys used as i18n keys under "modality.*" namespace; resolved // at render time inside `buildBackendOptions`. -const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'embeddings', 'reranker', 'detection', 'vad'] +const MODALITY_KEYS = ['text', 'asr', 'tts', 'image', 'video', 'embeddings', 'reranker', 'detection', 'vad'] // buildBackendOptions groups known backends by modality and tags // auto_detect=false entries with a muted "manual pick" badge so users diff --git a/core/http/react-ui/src/pages/VideoGen.jsx b/core/http/react-ui/src/pages/VideoGen.jsx index fbe43209e..d676b0fdc 100644 --- a/core/http/react-ui/src/pages/VideoGen.jsx +++ b/core/http/react-ui/src/pages/VideoGen.jsx @@ -8,10 +8,11 @@ import LoadingSpinner from '../components/LoadingSpinner' import GenerationProgress from '../components/GenerationProgress' import ErrorWithTraceLink from '../components/ErrorWithTraceLink' import MediaHistory from '../components/MediaHistory' +import MediaInput from '../components/biometrics/MediaInput' import { videoApi, fileToBase64 } from '../utils/api' import { useMediaHistory } from '../hooks/useMediaHistory' -const SIZES = ['256x256', '512x512', '768x768', '1024x1024'] +const SIZES = ['256x256', '512x512', '768x768', '1024x1024', '832x480', '1280x720'] export default function VideoGen() { const { model: urlModel } = useParams() @@ -31,9 +32,10 @@ export default function VideoGen() { const [error, setError] = useState(null) const [videos, setVideos] = useState([]) const [showAdvanced, setShowAdvanced] = useState(false) - const [showImageInputs, setShowImageInputs] = useState(false) + const [showMediaInputs, setShowMediaInputs] = useState(false) const [startImage, setStartImage] = useState(null) const [endImage, setEndImage] = useState(null) + const [audioInput, setAudioInput] = useState(null) const { addEntry, selectEntry, selectedEntry, historyProps } = useMediaHistory('video') const handleGenerate = async (e) => { @@ -55,6 +57,7 @@ export default function VideoGen() { if (cfgScale) body.cfg_scale = parseFloat(cfgScale) if (startImage) body.start_image = startImage if (endImage) body.end_image = endImage + if (audioInput?.base64) body.audio = audioInput.base64 try { const data = await videoApi.generate(body) @@ -116,24 +119,46 @@ export default function VideoGen() { -
setShowAdvanced(!showAdvanced)}> - {t('video.labels.advanced')} -
+