diff --git a/README.md b/README.md index 290378751..27c191d46 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native | Backend | What it does | |---------|-------------| -| [vllm.cpp](https://github.com/mudler/vllm.cpp) | From-scratch C++20 port of vLLM for text generation: paged KV cache, continuous batching, prefix caching, safetensors + GGUF loading, engine-enforced structured output, on CPU, CUDA, Metal and Vulkan | +| [vllm.cpp](https://github.com/mudler/vllm.cpp) | From-scratch C++20 port of vLLM for text generation: paged KV cache, continuous batching, prefix caching, safetensors + GGUF loading, engine-enforced structured output, on CPU, CUDA, Metal and Vulkan. Also serves MiniMax-H3 joint video+audio generation | | [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet ASR (tdt/ctc/rnnt/hybrid), with cache-aware streaming transcription | | [moss-transcribe.cpp](https://github.com/localai-org/moss-transcribe.cpp) | C++/GGML port of OpenMOSS MOSS-Transcribe-Diarize: joint long-form transcription, speaker diarization and timestamping in a single pass | | [moss-tts.cpp](https://github.com/mudler/moss-tts.cpp) | C++/GGML port of the OpenMOSS MOSS-TTS family: text-to-speech (MOSS-TTS-Local v1.5, 48 kHz stereo) with reference-audio voice cloning, through the MOSS-Audio-Tokenizer neural codec | diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index e8a2279bf..891b5e57c 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=0757cac231ecd571a83c4fd2f50805c9251fc225 +VLLM_CPP_VERSION?=2b08dd246e04b3f0a4bf1f276170fd28004ced01 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun @@ -106,13 +106,24 @@ else LIB=libvllm.so endif -sources/vllm.cpp: +# patches/ carries fixes the pinned engine SHA does not have yet. `git apply` +# is deliberately unguarded: a patch that no longer applies must FAIL the clone +# loudly, because the alternative is a pin that silently ships without a fix it +# is documented to carry. Each patch header says which pin retires it. +VLLM_CPP_PATCHES=$(wildcard patches/*.patch) + +sources/vllm.cpp: $(VLLM_CPP_PATCHES) + rm -rf sources/vllm.cpp mkdir -p sources/vllm.cpp cd sources/vllm.cpp && \ git init && \ git remote add origin $(VLLM_CPP_REPO) && \ git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \ - git checkout FETCH_HEAD + git checkout FETCH_HEAD && \ + for p in $(VLLM_CPP_PATCHES); do \ + echo "==> applying $$p"; \ + git apply ../../$$p || exit 1; \ + done ifeq ($(MLX_ENABLED),1) # A stamp FILE, not a phony target: a phony prerequisite is always "newer" than @@ -165,7 +176,7 @@ $(LIB): sources/vllm.cpp $(MLX_STAMP) cmake --build . --config Release -j$(JOBS) --target vllm_shared cp -fL build/$(LIB) ./$(LIB) -vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB) +vllm-cpp: main.go govllmcpp.go backend.go chat.go options.go video.go $(LIB) CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./ package: vllm-cpp diff --git a/backend/go/vllm-cpp/README.md b/backend/go/vllm-cpp/README.md index 74518636e..7bfb92241 100644 --- a/backend/go/vllm-cpp/README.md +++ b/backend/go/vllm-cpp/README.md @@ -1,12 +1,15 @@ # vllm-cpp backend -LocalAI text-generation backend for [vllm.cpp](https://github.com/mudler/vllm.cpp), +LocalAI backend for [vllm.cpp](https://github.com/mudler/vllm.cpp), the LocalAI-team C++20 port of vLLM (paged KV cache, continuous batching, safetensors + GGUF loading, CUDA / CPU / Metal / Vulkan) with no Python at inference time. +It serves two things: text generation, and MiniMax-H3 joint video+audio +generation. + The backend dlopens the engine's stable C ABI (`libvllm`, `include/vllm.h`, -ABI v10) through purego: +ABI v16) through purego: - `Load` -> `vllm_engine_load`: accepts a `.gguf` file or a HF-style model directory (`config.json` + safetensors). `context_size` maps to @@ -29,6 +32,12 @@ ABI v10) through purego: LocalAI's Go-side grammar-constrained tool calling; JSON-schema / regex / choice constraints are also exposed by the ABI. +`patches/` carries fixes the pinned engine SHA does not have yet, applied to +the clone the same way `longcat-video` patches its upstream. `git apply` is +unguarded on purpose: a patch that stops applying must fail the clone loudly +rather than leave a pin silently missing a fix it is documented to carry. Each +patch header says what retires it. + The struct mirrors in `govllmcpp.go` are hand-written against one ABI version, and the engine refuses to load against any other. Moving `VLLM_CPP_VERSION` in the Makefile therefore means updating `abiVersion` plus the mirrors (and their @@ -47,6 +56,68 @@ options: - max_num_seqs:16 ``` +## MiniMax-H3 video+audio generation + +`GenerateVideo` -> `vllm_video_generate` (ABI v12). H3 renders picture and sound +together, so the output MP4 carries a real AAC track. + +The video engine is a SECOND handle (`vllm_video_engine`), not a mode of the +text one, because H3 is a checkpoint SET rather than a model directory: the DiT, +the text encoder and two VAEs are separate artifacts, and vllm.cpp has the two +loaders refuse each other's checkpoints. `Load` takes the video branch when the +model config carries any of the video options below; `parameters.model` is the +DiT and everything else is named in `options:`. + +```yaml +name: minimax-h3-fl2va-q4 +backend: vllm-cpp +cuda: true +known_usecases: [video] +parameters: + model: minimax-h3/MiniMax-H3-FL2VA-Q4_K_M.gguf +options: +- video_encoder:minimax-h3/qwen3vl-32B-MiniMax-H3-Q4_K_M.gguf +- video_tokenizer:minimax-h3/tokenizer.json +- video_vae:minimax-h3/video_vae.safetensors +- video_vae_config:minimax-h3/video_vae_config.json +- audio_vae:minimax-h3/audio_vae.safetensors +- audio_vae_config:minimax-h3/audio_vae_config.json +- video_partition:fl2va +- video_device:cuda +- video_dequant_bf16:true +- video_width:1344 +- video_height:768 +- video_num_frames:124 +``` + +Three things are worth knowing before touching this path. + +**The partition is declared, not detected, and a mismatch does not fail +cleanly.** The FL2VA DiT serves `t2va` and `fl2va`; `ref2va` is a different +checkpoint. The community GGUF/NVFP4 quantisations strip the release metadata +and the two DiTs are byte-structurally identical, so the engine refuses every +generate until `video_partition` says which one it has. Handing reference +conditioning to an FL2VA DiT renders for hours and returns a coloured lattice +over the frame, so `checkPartitionConditioning` refuses that combination here, +before the engine is called. + +**ffmpeg comes from the host.** libvllm writes the frames and the WAV and +COMPOSES the mux argv, then spawns nothing — that process boundary is upstream's +decision. `muxVideo` takes the composed argv, substitutes `argv[0]` with the +resolved binary and execs it; the backend image is `FROM scratch` and carries no +ffmpeg, the same arrangement `vibevoice-cpp` uses for transcoding. ffmpeg also +converts a `start_image`/`end_image` upload into the binary PPM at the exact +output canvas the engine requires, since libvllm vendors neither an image codec +nor a resampler. + +**It is slow.** Roughly 176 s per denoise step at 1344x768 on a 20-SM device, so +the 50-step default is hours. Nothing here imposes a deadline. + +Geometry mirrors the engine so the two agree: the canvas is truncated onto a +32-pixel grid, the frame count sits on the 17n+5 grid, and an unspecified canvas +with a keyframe is derived from that image's aspect on a 768-pixel short edge +(`MiniMaxH3ResolveShape`, `minimax_h3_planner.cpp`). + ## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill) `BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index d82cc320a..5292c42ea 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -28,7 +28,12 @@ type VllmCpp struct { base.Base engine uintptr - opts loadOptions + // videoEngine is the MiniMax-H3 handle (ABI v12). It is deliberately a + // SECOND handle, not a mode of the first: H3 is a checkpoint set rather + // than a model directory, and vllm.cpp has the two loaders refuse each + // other's checkpoints. Exactly one of the two is ever non-zero. + videoEngine uintptr + opts loadOptions } // Stream registry: the per-request bridge between the C token callback and @@ -109,6 +114,14 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error { v.opts = parseOptions(opts) + // MiniMax-H3 is a checkpoint SET behind its own engine handle, so the + // branch is taken before any text-engine knob is resolved. The two loaders + // refuse each other's checkpoints, which is why this is decided from the + // config rather than probed. + if v.opts.video.engaged() { + return v.loadVideo(opts, model) + } + // A DFlash draft is a second checkpoint the engine opens by path, and the // engine never downloads one. Resolve it against LocalAI's models directory // now so a repo-id spelling works, and so a missing draft fails here with an @@ -194,6 +207,10 @@ func (v *VllmCpp) Free() error { vllmEngineFree(v.engine) v.engine = 0 } + if v.videoEngine != 0 { + vllmVideoEngineFree(v.videoEngine) + v.videoEngine = 0 + } return nil } diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index bca9944d4..5de1c0e64 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v16). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 10 +const abiVersion = 17 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -70,7 +70,15 @@ type cModelParams struct { SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) - _ [4]byte // trailing pad to the struct's 8-byte alignment + // v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and + // "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but + // the fields MUST be mirrored: the C side reads sizeof(vllm_model_params) + // bytes off the pointer we hand it, so a Go struct that stopped at + // EnableJumpForward would have vllm_engine_load read 24 bytes past our + // allocation and size the KV pool from whatever sat there. + Device int32 // 0 auto, 1 cpu, 2 cuda (ABI v14) + GPUMemoryUtil float64 // 0 => 0.92 (ABI v16) + KVCacheMemoryBytes int64 // 0 => unset (ABI v16) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). @@ -117,6 +125,79 @@ type cCompletion struct { CompletionTokens int32 } +// ── Video+audio generation (ABI v12, MiniMax-H3) ──────────────────────────── +// +// A video engine is a SEPARATE handle from vllm_engine: H3 is a checkpoint SET +// (DiT + text encoder + two VAEs), not one model directory, and the two loaders +// refuse each other's checkpoints on purpose. Offsets are asserted in +// video_test.go the same way the text PODs are in vllmcpp_test.go. + +// cVideoModelParams mirrors vllm_video_model_params. Nine pointers then three +// int32s, so only the trailing pad is implicit. +type cVideoModelParams struct { + DitPath uintptr // const char* + EncoderPath uintptr // const char* + TokenizerPath uintptr // const char* + VideoVaePath uintptr // const char* + VideoVaeConfigPath uintptr // const char* + AudioVaePath uintptr // const char* + AudioVaeConfigPath uintptr // const char* + PromptEmbedsPath uintptr // const char* + Partition uintptr // const char*; "fl2va" | "ref2va", REQUIRED + Device int32 // 0 cpu, 1 cuda + DequantBf16 int32 // 0 keep-quant, 1 dequant/stream bf16 + Fp4Resident int32 // NVFP4+cuda: keep FP4 packed, Marlin W4A16 + _ [4]byte // trailing pad to the struct's 8-byte alignment +} + +// cVideoParams mirrors vllm_video_params. `width`/`height` and `num_frames`/ +// `steps` pair up into 8-byte slots; the uint64 seed forces the alignment after +// them, and the float noise_aug leaves a pad before output_dir. +type cVideoParams struct { + Prompt uintptr // const char* + Width int32 + Height int32 + NumFrames int32 // <= 1 => per-task default (124 for t2va/fl2va) + Steps int32 // <= 0 => the H3 default (50) + Seed uint64 + HasSeed int32 + _ [4]byte + FirstFrame uintptr // const char*; fl2va keyframe, binary PPM (P6) + LastFrame uintptr // const char* + RefImage uintptr // const char*; ref2va only + RefVideo uintptr // const char*; ref2va only, a frame_%06d.ppm DIRECTORY + RefAudio uintptr // const char*; ref2va only, 16-bit PCM WAV + NoiseAug float32 // <= 0 => 1.0 + _ [4]byte + OutputDir uintptr // const char*; REQUIRED +} + +// cVideoResult mirrors vllm_video_result. Every member is library-allocated and +// released together by vllm_video_result_free. +type cVideoResult struct { + FrameDir uintptr // char*, holds frame_%06d.ppm + AudioPath uintptr // char*, 16-bit PCM WAV + FrameCount int32 + Width int32 + Height int32 + Fps int32 + SampleRate int32 + _ [4]byte + MuxArgv uintptr // char**, NULL-terminated at MuxArgc + MuxArgc int32 + _ [4]byte +} + +// cVideoMuxParams mirrors vllm_video_mux_params. The library composes the argv; +// spawning it is the CALLER's job, which is why no ffmpeg lives in libvllm. +type cVideoMuxParams struct { + Frames uintptr // const char*; printf pattern, dir/frame_%06d.ppm + AudioPath uintptr // const char*; NULL/empty => a silent clip + OutputPath uintptr // const char*; the .mp4 to write + Fps int32 // <= 0 => the H3 default (24) + Crf int32 // <= 0 => the library default (18) +} + // defaultSamplingParams mirrors vllm_sampling_params_default(). func defaultSamplingParams() cSamplingParams { return cSamplingParams{ @@ -148,6 +229,14 @@ var ( vllmLastError func() string vllmVersion func() string vllmABIVersion func() int32 + + // Video+audio generation (ABI v12). + vllmVideoEngineLoad func(params, out unsafe.Pointer) int32 + vllmVideoEngineFree func(engine uintptr) + vllmVideoGenerate func(engine uintptr, params, out unsafe.Pointer) int32 + vllmVideoResultFree func(out unsafe.Pointer) + vllmVideoMuxArgv func(params, outArgv, outArgc unsafe.Pointer) int32 + vllmVideoMuxArgvFre func(argv uintptr, argc int32) ) type libFunc struct { @@ -175,6 +264,12 @@ func registerLib(libName string) error { {&vllmLastError, "vllm_last_error"}, {&vllmVersion, "vllm_version"}, {&vllmABIVersion, "vllm_abi_version"}, + {&vllmVideoEngineLoad, "vllm_video_engine_load"}, + {&vllmVideoEngineFree, "vllm_video_engine_free"}, + {&vllmVideoGenerate, "vllm_video_generate"}, + {&vllmVideoResultFree, "vllm_video_result_free"}, + {&vllmVideoMuxArgv, "vllm_video_mux_argv"}, + {&vllmVideoMuxArgvFre, "vllm_video_mux_argv_free"}, } { purego.RegisterLibFunc(lf.ptr, lib, lf.name) } @@ -222,3 +317,19 @@ func goString(p uintptr) string { } return string(unsafe.Slice((*byte)(base), n)) } + +// goStringSlice copies a C `char*` array of n entries. Used for the ffmpeg argv +// the library composes: it is copied out immediately so the caller can free the +// C allocation before ever spawning the process. +func goStringSlice(p uintptr, n int32) []string { + if p == 0 || n <= 0 { + return nil + } + //nolint:govet // C-owned pointer handed over by purego, valid for this call + entries := unsafe.Slice((**byte)(unsafe.Pointer(p)), int(n)) // #nosec G103 -- C-owned, copied out immediately + out := make([]string, 0, n) + for _, e := range entries { + out = append(out, goString(uintptr(unsafe.Pointer(e)))) // #nosec G103 -- ditto + } + return out +} diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index 0cdf4dd4c..86717e7ac 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -62,6 +62,66 @@ type loadOptions struct { // Override for the tokenizer_config.json the chat template is read from // (ABI v9). Empty = /tokenizer_config.json. tokenizerConfigPath string + // MiniMax-H3 video+audio generation (ABI v12). Present only when the config + // carries at least one of its keys; see videoOptions.engaged. + video videoOptions +} + +// videoOptions is the MiniMax-H3 checkpoint SET plus its generation defaults. +// +// H3 is not one model directory: the DiT, the text encoder and the two VAEs are +// separate artifacts, which is why vllm.cpp gives video its own engine handle +// (vllm_video_engine, ABI v12) rather than another vllm_engine. The DiT is the +// model config's `parameters.model`; everything else arrives through these +// options, so one gallery entry can name five files. +// +// The geometry/frame defaults exist because H3's trained canvas is nothing like +// the generic /video defaults: 1344x768 at 124 frames is a ~5.2 s clip, and the +// frame count must sit on the 17n+5 grid. A request that leaves a field unset +// gets the model's own default from here instead of a canvas the checkpoint was +// never trained at. +type videoOptions struct { + encoderPath string // H3-Encoder GGUF or bf16 shard dir + tokenizerPath string // tokenizer.json, needed with an encoder + videoVaePath string + videoVaeConfig string + audioVaePath string + audioVaeConfig string + promptEmbedsPath string // fallback conditioning when there is no encoder + // The served checkpoint PARTITION. Community GGUF/NVFP4 files strip the + // release metadata and the FL2VA/Ref2VA DiTs are byte-structurally + // identical, so the engine refuses every generate until it is DECLARED. + // "fl2va" serves t2va + fl2va; "ref2va" serves reference conditioning. + partition string + device int32 // 0 cpu, 1 cuda (the ABI's own encoding, no auto slot) + deviceSet bool + dequantBf16 int32 + fp4Resident int32 + // Per-model generation defaults, applied when the request leaves the field + // at 0. + width int32 + height int32 + numFrames int32 + steps int32 + // Where frames + WAV are written. Empty = a temporary directory beside the + // requested output, removed once the mux succeeds. Set it to keep the + // frame_%06d.ppm runs around (they are what ref2va's ref_video consumes). + workdir string + // The ffmpeg binary the composed mux argv is exec'd with. Empty = "ffmpeg" + // from PATH. libvllm composes the argv and spawns nothing, by design. + ffmpeg string + crf int32 +} + +// engaged reports whether this config describes an H3 video engine. Load uses +// it to choose which of the two mutually exclusive engine handles to open: the +// checkpoints refuse each other, so guessing is not an option, and every key +// below is meaningless to the text engine. +func (v videoOptions) engaged() bool { + return v.encoderPath != "" || v.tokenizerPath != "" || + v.videoVaePath != "" || v.videoVaeConfig != "" || + v.audioVaePath != "" || v.audioVaeConfig != "" || + v.promptEmbedsPath != "" || v.partition != "" } func parseOptions(opts *pb.ModelOptions) loadOptions { @@ -110,10 +170,94 @@ func applyOptionsList(lo *loadOptions, options []string) { if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil { lo.enableJumpForward = boolTriState(b) } + default: + applyVideoOption(&lo.video, strings.TrimSpace(k), v) } } } +// applyVideoOption reads one MiniMax-H3 key. Split out of applyOptionsList so +// the video surface stays legible next to the videoOptions it fills, and so +// video_test.go can exercise it directly. +func applyVideoOption(vo *videoOptions, key, value string) bool { + v := strings.TrimSpace(value) + switch key { + case "video_encoder": + vo.encoderPath = v + case "video_tokenizer": + vo.tokenizerPath = v + case "video_vae": + vo.videoVaePath = v + case "video_vae_config": + vo.videoVaeConfig = v + case "audio_vae": + vo.audioVaePath = v + case "audio_vae_config": + vo.audioVaeConfig = v + case "video_prompt_embeds": + vo.promptEmbedsPath = v + case "video_partition": + vo.partition = strings.ToLower(v) + case "video_device": + switch strings.ToLower(v) { + case "cpu": + vo.device, vo.deviceSet = videoDeviceCPU, true + case "cuda", "gpu": + vo.device, vo.deviceSet = videoDeviceCUDA, true + default: + xlog.Warn("[vllm-cpp] ignoring unknown video_device", "value", v) + } + case "video_dequant_bf16": + if b, err := strconv.ParseBool(v); err == nil { + vo.dequantBf16 = boolInt32(b) + } + case "video_fp4_resident": + if b, err := strconv.ParseBool(v); err == nil { + vo.fp4Resident = boolInt32(b) + } + case "video_width": + vo.width = parseInt32(v, vo.width) + case "video_height": + vo.height = parseInt32(v, vo.height) + case "video_num_frames": + vo.numFrames = parseInt32(v, vo.numFrames) + case "video_steps": + vo.steps = parseInt32(v, vo.steps) + case "video_workdir": + vo.workdir = v + case "video_crf": + vo.crf = parseInt32(v, vo.crf) + case "ffmpeg", "ffmpeg_path": + vo.ffmpeg = v + default: + return false + } + return true +} + +// videoScalarString renders an engine_args scalar so the video keys can share +// one parser with the "key:value" list. Objects and arrays have no video +// meaning and are left to the caller's unknown-key path. +func videoScalarString(v any) (string, bool) { + switch t := v.(type) { + case string: + return t, true + case bool: + return strconv.FormatBool(t), true + case float64: + return strconv.FormatFloat(t, 'f', -1, 64), true + default: + return "", false + } +} + +func boolInt32(b bool) int32 { + if b { + return 1 + } + return 0 +} + // applyEngineArgs overlays the `engine_args:` JSON object. A document that does // not parse is logged and skipped: engine_args is shared with the other engines // (the vLLM and SGLang backends read the same field), so a stray key must not @@ -160,6 +304,9 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) { lo.enableJumpForward = boolTriState(b) } default: + if s, ok := videoScalarString(v); ok && applyVideoOption(&lo.video, k, s) { + continue + } xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k) } } diff --git a/backend/go/vllm-cpp/video.go b/backend/go/vllm-cpp/video.go new file mode 100644 index 000000000..1b2777087 --- /dev/null +++ b/backend/go/vllm-cpp/video.go @@ -0,0 +1,634 @@ +package main + +// MiniMax-H3 video+audio generation over the vllm.cpp C ABI (v12). +// +// Two things make this different from the text path, and both come from the +// engine's own shape rather than from LocalAI: +// +// 1. A video engine is loaded from a checkpoint SET - the DiT, the text +// encoder and two VAEs are separate artifacts - so it is its own handle +// (vllm_video_engine) and its own Load branch. The two loaders refuse each +// other's checkpoints on purpose. +// 2. libvllm writes frames + a WAV and COMPOSES the ffmpeg argv, but spawns +// nothing. That process boundary is deliberate upstream, so the mux lives +// here: we take the composed argv, substitute argv[0], and exec it. ffmpeg +// comes from PATH the same way the vibevoice-cpp backend takes it. +// +// Generation is SLOW - roughly 176 s per denoise step at 1344x768 on a 20-SM +// device, so a default 50-step render is hours, not seconds. Nothing here +// imposes a deadline: GenerateVideo blocks for as long as the engine needs and +// the gRPC call carries LocalAI's application context. + +import ( + "fmt" + "image" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "unsafe" + + // Registered for image.DecodeConfig only: a staged keyframe arrives as + // whatever the caller uploaded, and we need its geometry to size the canvas. + _ "image/gif" + _ "image/jpeg" + _ "image/png" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" +) + +// vllm_video_model_params.device (vllm.h): no auto slot, unlike the text +// engine's v14 device field. +const ( + videoDeviceCPU int32 = 0 + videoDeviceCUDA int32 = 1 +) + +// H3's shipped geometry. The canvas is truncated onto a 32-pixel grid and the +// frame count onto the 17n+5 grid by the engine itself +// (MiniMaxH3ResolveShape / MiniMaxH3AlignFrameCount in +// src/vllm/model_executor/models/minimax_h3_planner.cpp); mirrored here only so +// a keyframe can be resampled to the exact canvas the engine will render at. +const ( + h3CanvasMultiple int32 = 32 + h3FrameGrid int32 = 17 + h3FrameOffset int32 = 5 + h3ShortEdge int32 = 768 +) + +// videoPartitions are the two DECLARED partitions of the H3 release. The FL2VA +// checkpoint serves t2va and fl2va; ref2va is a different checkpoint. Passing +// reference conditioning against an fl2va DiT is a partition mismatch that +// renders a coloured lattice over the frame rather than failing cleanly, which +// is why it is refused here before the engine is ever called. +const ( + partitionFL2VA = "fl2va" + partitionRef2VA = "ref2va" +) + +// videoRequestParams are the per-request `params` keys this backend accepts. +// Unknown keys are an error rather than a silent drop: a misspelled reference +// path would otherwise produce a perfectly successful render of the wrong +// thing, hours later. +var videoRequestParams = []string{"noise_aug", "ref_image", "ref_video", "crf"} + +// loadVideo opens the H3 checkpoint set. `dit` is the model config's +// parameters.model; every other artifact comes from the options. +func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error { + vo := &v.opts.video + + // Relative option paths resolve against LocalAI's models directory, which + // is where the gallery lands the five H3 files. + resolve := func(p string) string { + if p == "" || filepath.IsAbs(p) || opts.ModelPath == "" { + return p + } + return filepath.Join(opts.ModelPath, p) + } + vo.encoderPath = resolve(vo.encoderPath) + vo.tokenizerPath = resolve(vo.tokenizerPath) + vo.videoVaePath = resolve(vo.videoVaePath) + vo.videoVaeConfig = resolve(vo.videoVaeConfig) + vo.audioVaePath = resolve(vo.audioVaePath) + vo.audioVaeConfig = resolve(vo.audioVaeConfig) + vo.promptEmbedsPath = resolve(vo.promptEmbedsPath) + vo.workdir = resolve(vo.workdir) + + // A VAE config carries the per-channel latents_mean/latents_std and the + // temporal clip_length/token_drop; decode is wrong without it. The release + // ships it beside the weights, so default to that rather than making every + // config repeat it. + if vo.videoVaeConfig == "" && vo.videoVaePath != "" { + vo.videoVaeConfig = siblingConfigJSON(vo.videoVaePath) + } + if vo.audioVaeConfig == "" && vo.audioVaePath != "" { + vo.audioVaeConfig = siblingConfigJSON(vo.audioVaePath) + } + + if vo.partition == "" { + // The community GGUF/NVFP4 quantisations strip the release metadata and + // the two DiTs are byte-structurally identical, so the engine cannot + // infer this and refuses every generate until it is declared. The + // shipped FL2VA checkpoint is the one the gallery entry installs. + vo.partition = partitionFL2VA + xlog.Warn("[vllm-cpp] video partition not declared, assuming the FL2VA checkpoint", + "hint", "set options: [video_partition:fl2va] or [video_partition:ref2va] to match the DiT you installed") + } + if vo.partition != partitionFL2VA && vo.partition != partitionRef2VA { + return fmt.Errorf("vllm-cpp: video_partition must be %q or %q, got %q", + partitionFL2VA, partitionRef2VA, vo.partition) + } + if vo.videoVaePath == "" || vo.audioVaePath == "" { + return fmt.Errorf("vllm-cpp: MiniMax-H3 needs both VAEs: set options: " + + "[video_vae: