feat(vllm-cpp): add video_lora_dir option for runtime prompt-activated LoRA (#12119)

Add a video_lora_dir model option that passes lora_dir as a model-load
extra to the vllm.cpp engine. The engine uses it to resolve
<lora:name:strength> prompt tags at request time, loading the safetensors
LoRA factors and applying per-request deltas without touching base weights
(row ROAD-V1-LORA-RUNTIME).

This is distinct from the existing load-time lora_path/lora_strength fusion
(ROAD-V1-DIT-LORA), which bakes deltas into the DiT weights at load. Both
mechanisms coexist: load-time adapters are always active, while prompt-tag
adapters are selected per request.

The prompt passes through verbatim — the engine strips the tags internally.

FOLLOWING_AGENTS_PROTOCOL

Following-Agents-Protocol: true
AI-Assisted: true
Assisted-by: AGENT:regolo/glm5.2 [maki]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/go/vllm-cpp/options.go

@@ -103,6 +103,12 @@ type videoOptions struct {
 	height    int32
 	numFrames int32
 	steps     int32
+	// Directory for runtime prompt-activated LoRA. The engine resolves
+	// <lora:name:strength> prompt tags against safetensors files in this
+	// directory at request time (row ROAD-V1-LORA-RUNTIME). Distinct from the
+	// load-time lora_path/lora_strength fusion, which bakes deltas into the
+	// weights at load.
+	loraDir string
 	// Where frames + WAV are written. Empty = a temporary directory beside the
 	// requested output, removed once the mux succeeds. Set it to keep the
 	// frame_%06d.ppm runs around (they are what ref2va's ref_video consumes).
@@ -258,6 +264,8 @@ func applyVideoOption(vo *videoOptions, key, value string) bool {
 		vo.workdir = v
 	case "video_crf":
 		vo.crf = parseInt32(v, vo.crf)
+	case "video_lora_dir":
+		vo.loraDir = v
 	case "ffmpeg", "ffmpeg_path":
 		vo.ffmpeg = v
 	default:

backend/go/vllm-cpp/video.go

@@ -168,6 +168,15 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error {
 		opts.GetLoraAdapters(), opts.GetLoraScales(),
 		opts.GetLoraAdapter(), opts.GetLoraScale(),
 		opts.ModelPath)
+	loraN := len(extraKeys) / 2 // each load-time adapter = lora_path + lora_strength
+	// Runtime prompt-activated LoRA: lora_dir tells the engine where to
+	// resolve <lora:name:strength> prompt tags at request time (row
+	// ROAD-V1-LORA-RUNTIME). The engine strips the tags from the prompt and
+	// applies the deltas per-request without touching base weights.
+	if vo.loraDir != "" {
+		extraKeys = append(extraKeys, "lora_dir")
+		extraValues = append(extraValues, vo.loraDir)
+	}
 	if len(extraKeys) > 0 {
 		keyPtrs, keyBacking := cStringArray(extraKeys)
 		valPtrs, valBacking := cStringArray(extraValues)
@@ -182,7 +191,7 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error {
 		"videoVae", vo.videoVaePath, "audioVae", vo.audioVaePath,
 		"partition", vo.partition, "device", videoDeviceName(vo.device),
 		"dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1,
-		"loraAdapters", len(extraKeys)/2)
+		"loraAdapters", loraN, "loraDir", vo.loraDir)
 
 	var engine uintptr
 	rc := vllmVideoEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params

backend/go/vllm-cpp/video_test.go

@@ -339,6 +339,26 @@ var _ = Describe("buildLoraExtras", func() {
 	})
 })
 
+var _ = Describe("video_lora_dir option", func() {
+	It("parses video_lora_dir into loraDir", func() {
+		vo := videoOptions{}
+		Expect(applyVideoOption(&vo, "video_lora_dir", "/data/loras")).To(BeTrue())
+		Expect(vo.loraDir).To(Equal("/data/loras"))
+	})
+
+	It("trims whitespace around the path", func() {
+		vo := videoOptions{}
+		Expect(applyVideoOption(&vo, "video_lora_dir", "  /data/loras  ")).To(BeTrue())
+		Expect(vo.loraDir).To(Equal("/data/loras"))
+	})
+
+	It("returns false for an unknown key", func() {
+		vo := videoOptions{}
+		Expect(applyVideoOption(&vo, "video_not_a_real_key", "x")).To(BeFalse())
+		Expect(vo.loraDir).To(BeEmpty())
+	})
+})
+
 // writePPM writes a valid P6 header of the given geometry. Only the header is
 // read by anything under test, so the pixel payload is left off.
 func writePPM(width, height int) string {