From 3d63736c56beec95f0a4fcb1d57c0168032706bf Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Fri, 18 Sep 2026 10:36:36 +0200 Subject: [PATCH] feat(vllm-cpp): add video_lora_dir option for runtime prompt-activated LoRA (#12119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a video_lora_dir model option that passes lora_dir as a model-load extra to the vllm.cpp engine. The engine uses it to resolve prompt tags at request time, loading the safetensors LoRA factors and applying per-request deltas without touching base weights (row ROAD-V1-LORA-RUNTIME). This is distinct from the existing load-time lora_path/lora_strength fusion (ROAD-V1-DIT-LORA), which bakes deltas into the DiT weights at load. Both mechanisms coexist: load-time adapters are always active, while prompt-tag adapters are selected per request. The prompt passes through verbatim — the engine strips the tags internally. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:regolo/glm5.2 [maki] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- backend/go/vllm-cpp/options.go | 8 ++++++++ backend/go/vllm-cpp/video.go | 11 ++++++++++- backend/go/vllm-cpp/video_test.go | 20 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index b75e5a7fc..7e3edf4d3 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -103,6 +103,12 @@ type videoOptions struct { height int32 numFrames int32 steps int32 + // Directory for runtime prompt-activated LoRA. The engine resolves + // prompt tags against safetensors files in this + // directory at request time (row ROAD-V1-LORA-RUNTIME). Distinct from the + // load-time lora_path/lora_strength fusion, which bakes deltas into the + // weights at load. + loraDir string // Where frames + WAV are written. Empty = a temporary directory beside the // requested output, removed once the mux succeeds. Set it to keep the // frame_%06d.ppm runs around (they are what ref2va's ref_video consumes). @@ -258,6 +264,8 @@ func applyVideoOption(vo *videoOptions, key, value string) bool { vo.workdir = v case "video_crf": vo.crf = parseInt32(v, vo.crf) + case "video_lora_dir": + vo.loraDir = v case "ffmpeg", "ffmpeg_path": vo.ffmpeg = v default: diff --git a/backend/go/vllm-cpp/video.go b/backend/go/vllm-cpp/video.go index ba207e019..f9506a033 100644 --- a/backend/go/vllm-cpp/video.go +++ b/backend/go/vllm-cpp/video.go @@ -168,6 +168,15 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error { opts.GetLoraAdapters(), opts.GetLoraScales(), opts.GetLoraAdapter(), opts.GetLoraScale(), opts.ModelPath) + loraN := len(extraKeys) / 2 // each load-time adapter = lora_path + lora_strength + // Runtime prompt-activated LoRA: lora_dir tells the engine where to + // resolve prompt tags at request time (row + // ROAD-V1-LORA-RUNTIME). The engine strips the tags from the prompt and + // applies the deltas per-request without touching base weights. + if vo.loraDir != "" { + extraKeys = append(extraKeys, "lora_dir") + extraValues = append(extraValues, vo.loraDir) + } if len(extraKeys) > 0 { keyPtrs, keyBacking := cStringArray(extraKeys) valPtrs, valBacking := cStringArray(extraValues) @@ -182,7 +191,7 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error { "videoVae", vo.videoVaePath, "audioVae", vo.audioVaePath, "partition", vo.partition, "device", videoDeviceName(vo.device), "dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1, - "loraAdapters", len(extraKeys)/2) + "loraAdapters", loraN, "loraDir", vo.loraDir) var engine uintptr rc := vllmVideoEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params diff --git a/backend/go/vllm-cpp/video_test.go b/backend/go/vllm-cpp/video_test.go index 9f87cba03..418d2b356 100644 --- a/backend/go/vllm-cpp/video_test.go +++ b/backend/go/vllm-cpp/video_test.go @@ -339,6 +339,26 @@ var _ = Describe("buildLoraExtras", func() { }) }) +var _ = Describe("video_lora_dir option", func() { + It("parses video_lora_dir into loraDir", func() { + vo := videoOptions{} + Expect(applyVideoOption(&vo, "video_lora_dir", "/data/loras")).To(BeTrue()) + Expect(vo.loraDir).To(Equal("/data/loras")) + }) + + It("trims whitespace around the path", func() { + vo := videoOptions{} + Expect(applyVideoOption(&vo, "video_lora_dir", " /data/loras ")).To(BeTrue()) + Expect(vo.loraDir).To(Equal("/data/loras")) + }) + + It("returns false for an unknown key", func() { + vo := videoOptions{} + Expect(applyVideoOption(&vo, "video_not_a_real_key", "x")).To(BeFalse()) + Expect(vo.loraDir).To(BeEmpty()) + }) +}) + // writePPM writes a valid P6 header of the given geometry. Only the header is // read by anything under test, so the pixel payload is left off. func writePPM(width, height int) string {