[vllm-cpp] wire lora_adapters/lora_scales into DiT load-time fusion (#12082)

feat(vllm-cpp): wire lora_adapters/lora_scales into DiT load-time fusion

The vllm-cpp video backend now converts the standard LocalAI
lora_adapters/lora_scales config fields into indexed lora_path/
lora_strength extras pairs that vllm.cpp's ResolveDitLoraSpecs
consumes (row ROAD-V1-DIT-LORA).

The singular lora_adapter/lora_scale fields are folded in as the
first adapter. Relative paths resolve against the model directory.
Strength defaults to 1.0 when no scale is given. Adapters are fused
into DiT weights at load, so they are always loaded — no per-request
activation in this path.

Six unit tests cover the buildLoraExtras helper.

FOLLOWING_AGENTS_PROTOCOL

Following-Agents-Protocol: true
AI-Assisted: true
Assisted-by: AGENT:regolo/glm5.2 [TOOL]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/go/vllm-cpp/video.go

@@ -140,7 +140,7 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error {
 		DequantBf16: vo.dequantBf16,
 		Fp4Resident: vo.fp4Resident,
 	}
-	var keep [][]byte
+	var keep []any
 	setStr := func(dst *uintptr, s string) {
 		if s == "" {
 			return
@@ -159,11 +159,30 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error {
 	setStr(&mp.PromptEmbedsPath, vo.promptEmbedsPath)
 	setStr(&mp.Partition, vo.partition)
 
+	// Load-time LoRA fusion: the standard LocalAI lora_adapters/lora_scales
+	// config fields carry into the vllm.cpp extras seam as indexed
+	// lora_path/lora_strength pairs (row ROAD-V1-DIT-LORA). The engine fuses
+	// the deltas into the DiT weights at load, so the adapters are "always
+	// loaded" — there is no per-request activation in this path.
+	extraKeys, extraValues := buildLoraExtras(
+		opts.GetLoraAdapters(), opts.GetLoraScales(),
+		opts.GetLoraAdapter(), opts.GetLoraScale(),
+		opts.ModelPath)
+	if len(extraKeys) > 0 {
+		keyPtrs, keyBacking := cStringArray(extraKeys)
+		valPtrs, valBacking := cStringArray(extraValues)
+		mp.NExtras = int32(len(extraKeys))
+		mp.ExtraKeys = uintptr(unsafe.Pointer(&keyPtrs[0]))   // #nosec G103 -- borrowed by C for the load call only
+		mp.ExtraValues = uintptr(unsafe.Pointer(&valPtrs[0])) // #nosec G103 -- borrowed by C for the load call only
+		keep = append(keep, keyPtrs, valPtrs, keyBacking, valBacking)
+	}
+
 	xlog.Info("[vllm-cpp] Load (MiniMax-H3 video)", "dit", dit, "engine", vllmVersion(),
 		"encoder", vo.encoderPath, "tokenizer", vo.tokenizerPath,
 		"videoVae", vo.videoVaePath, "audioVae", vo.audioVaePath,
 		"partition", vo.partition, "device", videoDeviceName(vo.device),
-		"dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1)
+		"dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1,
+		"loraAdapters", len(extraKeys)/2)
 
 	var engine uintptr
 	rc := vllmVideoEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params
@@ -632,3 +651,41 @@ func siblingConfigJSON(weights string) string {
 	}
 	return candidate
 }
+
+// buildLoraExtras converts the standard LocalAI lora_adapters/lora_scales
+// config fields into the indexed lora_path/lora_strength extras pairs that
+// vllm.cpp's ResolveDitLoraSpecs expects (row ROAD-V1-DIT-LORA).
+//
+// The singular lora_adapter/lora_scale fields are folded in as the first
+// adapter. Adapter paths are resolved against modelPath when relative.
+// Strength defaults to 1.0 when no scale is given for an adapter.
+//
+// Returns nil, nil when no adapters are configured.
+func buildLoraExtras(adapters []string, scales []float32, singularAdapter string, singularScale float32, modelPath string) (keys, values []string) {
+	if singularAdapter != "" {
+		adapters = append([]string{singularAdapter}, adapters...)
+		scales = append([]float32{singularScale}, scales...)
+	}
+	if len(adapters) == 0 {
+		return nil, nil
+	}
+	for i, adapter := range adapters {
+		path := adapter
+		if !filepath.IsAbs(path) && modelPath != "" {
+			path = filepath.Join(modelPath, path)
+		}
+		suffix := ""
+		if i > 0 {
+			suffix = "_" + strconv.Itoa(i + 1)
+		}
+		keys = append(keys, "lora_path"+suffix)
+		values = append(values, path)
+		strength := 1.0
+		if i < len(scales) {
+			strength = float64(scales[i])
+		}
+		keys = append(keys, "lora_strength"+suffix)
+		values = append(values, strconv.FormatFloat(strength, 'f', -1, 32))
+	}
+	return keys, values
+}

backend/go/vllm-cpp/video_test.go

@@ -282,6 +282,63 @@ var _ = Describe("GenerateVideo preconditions", func() {
 	})
 })
 
+var _ = Describe("buildLoraExtras", func() {
+	It("returns nil when no adapters are configured", func() {
+		keys, vals := buildLoraExtras(nil, nil, "", 0, "/models")
+		Expect(keys).To(BeNil())
+		Expect(vals).To(BeNil())
+	})
+
+	It("builds indexed lora_path/lora_strength pairs for one adapter", func() {
+		keys, vals := buildLoraExtras(
+			[]string{"/abs/lora.safetensors"}, []float32{0.7}, "", 0, "/models")
+		Expect(keys).To(Equal([]string{"lora_path", "lora_strength"}))
+		Expect(vals).To(Equal([]string{"/abs/lora.safetensors", "0.7"}))
+	})
+
+	It("appends _N suffix from the second adapter onward", func() {
+		keys, vals := buildLoraExtras(
+			[]string{"/a.safetensors", "/b.safetensors"},
+			[]float32{0.5, 0.3}, "", 0, "")
+		Expect(keys).To(Equal([]string{
+			"lora_path", "lora_strength",
+			"lora_path_2", "lora_strength_2",
+		}))
+		Expect(vals).To(Equal([]string{
+			"/a.safetensors", "0.5",
+			"/b.safetensors", "0.3",
+		}))
+	})
+
+	It("defaults strength to 1 when no scale is given", func() {
+		keys, vals := buildLoraExtras(
+			[]string{"/a.safetensors"}, nil, "", 0, "")
+		Expect(keys).To(Equal([]string{"lora_path", "lora_strength"}))
+		Expect(vals).To(Equal([]string{"/a.safetensors", "1"}))
+	})
+
+	It("folds the singular lora_adapter/lora_scale in as the first adapter", func() {
+		keys, vals := buildLoraExtras(
+			[]string{"/second.safetensors"}, []float32{0.2},
+			"/first.safetensors", 0.8, "")
+		Expect(keys).To(Equal([]string{
+			"lora_path", "lora_strength",
+			"lora_path_2", "lora_strength_2",
+		}))
+		Expect(vals).To(Equal([]string{
+			"/first.safetensors", "0.8",
+			"/second.safetensors", "0.2",
+		}))
+	})
+
+	It("resolves relative adapter paths against the models directory", func() {
+		keys, vals := buildLoraExtras(
+			[]string{"loras/style.safetensors"}, nil, "", 0, "/models")
+		Expect(keys).To(Equal([]string{"lora_path", "lora_strength"}))
+		Expect(vals[0]).To(Equal(filepath.Join("/models", "loras/style.safetensors")))
+	})
+})
+
 // writePPM writes a valid P6 header of the given geometry. Only the header is
 // read by anything under test, so the pixel payload is left off.
 func writePPM(width, height int) string {