From 60679e5de65ca618a4b1133119a6a52967320e40 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 16 Sep 2026 19:39:16 +0200 Subject: [PATCH] [vllm-cpp] wire lora_adapters/lora_scales into DiT load-time fusion (#12082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(vllm-cpp): wire lora_adapters/lora_scales into DiT load-time fusion The vllm-cpp video backend now converts the standard LocalAI lora_adapters/lora_scales config fields into indexed lora_path/ lora_strength extras pairs that vllm.cpp's ResolveDitLoraSpecs consumes (row ROAD-V1-DIT-LORA). The singular lora_adapter/lora_scale fields are folded in as the first adapter. Relative paths resolve against the model directory. Strength defaults to 1.0 when no scale is given. Adapters are fused into DiT weights at load, so they are always loaded — no per-request activation in this path. Six unit tests cover the buildLoraExtras helper. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:regolo/glm5.2 [TOOL] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- backend/go/vllm-cpp/video.go | 61 ++++++++++++++++++++++++++++++- backend/go/vllm-cpp/video_test.go | 57 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/backend/go/vllm-cpp/video.go b/backend/go/vllm-cpp/video.go index 1b2777087..ba207e019 100644 --- a/backend/go/vllm-cpp/video.go +++ b/backend/go/vllm-cpp/video.go @@ -140,7 +140,7 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error { DequantBf16: vo.dequantBf16, Fp4Resident: vo.fp4Resident, } - var keep [][]byte + var keep []any setStr := func(dst *uintptr, s string) { if s == "" { return @@ -159,11 +159,30 @@ func (v *VllmCpp) loadVideo(opts *pb.ModelOptions, dit string) error { setStr(&mp.PromptEmbedsPath, vo.promptEmbedsPath) setStr(&mp.Partition, vo.partition) + // Load-time LoRA fusion: the standard LocalAI lora_adapters/lora_scales + // config fields carry into the vllm.cpp extras seam as indexed + // lora_path/lora_strength pairs (row ROAD-V1-DIT-LORA). The engine fuses + // the deltas into the DiT weights at load, so the adapters are "always + // loaded" — there is no per-request activation in this path. + extraKeys, extraValues := buildLoraExtras( + opts.GetLoraAdapters(), opts.GetLoraScales(), + opts.GetLoraAdapter(), opts.GetLoraScale(), + opts.ModelPath) + if len(extraKeys) > 0 { + keyPtrs, keyBacking := cStringArray(extraKeys) + valPtrs, valBacking := cStringArray(extraValues) + mp.NExtras = int32(len(extraKeys)) + mp.ExtraKeys = uintptr(unsafe.Pointer(&keyPtrs[0])) // #nosec G103 -- borrowed by C for the load call only + mp.ExtraValues = uintptr(unsafe.Pointer(&valPtrs[0])) // #nosec G103 -- borrowed by C for the load call only + keep = append(keep, keyPtrs, valPtrs, keyBacking, valBacking) + } + xlog.Info("[vllm-cpp] Load (MiniMax-H3 video)", "dit", dit, "engine", vllmVersion(), "encoder", vo.encoderPath, "tokenizer", vo.tokenizerPath, "videoVae", vo.videoVaePath, "audioVae", vo.audioVaePath, "partition", vo.partition, "device", videoDeviceName(vo.device), - "dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1) + "dequantBf16", vo.dequantBf16 == 1, "fp4Resident", vo.fp4Resident == 1, + "loraAdapters", len(extraKeys)/2) var engine uintptr rc := vllmVideoEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params @@ -632,3 +651,41 @@ func siblingConfigJSON(weights string) string { } return candidate } + +// buildLoraExtras converts the standard LocalAI lora_adapters/lora_scales +// config fields into the indexed lora_path/lora_strength extras pairs that +// vllm.cpp's ResolveDitLoraSpecs expects (row ROAD-V1-DIT-LORA). +// +// The singular lora_adapter/lora_scale fields are folded in as the first +// adapter. Adapter paths are resolved against modelPath when relative. +// Strength defaults to 1.0 when no scale is given for an adapter. +// +// Returns nil, nil when no adapters are configured. +func buildLoraExtras(adapters []string, scales []float32, singularAdapter string, singularScale float32, modelPath string) (keys, values []string) { + if singularAdapter != "" { + adapters = append([]string{singularAdapter}, adapters...) + scales = append([]float32{singularScale}, scales...) + } + if len(adapters) == 0 { + return nil, nil + } + for i, adapter := range adapters { + path := adapter + if !filepath.IsAbs(path) && modelPath != "" { + path = filepath.Join(modelPath, path) + } + suffix := "" + if i > 0 { + suffix = "_" + strconv.Itoa(i + 1) + } + keys = append(keys, "lora_path"+suffix) + values = append(values, path) + strength := 1.0 + if i < len(scales) { + strength = float64(scales[i]) + } + keys = append(keys, "lora_strength"+suffix) + values = append(values, strconv.FormatFloat(strength, 'f', -1, 32)) + } + return keys, values +} diff --git a/backend/go/vllm-cpp/video_test.go b/backend/go/vllm-cpp/video_test.go index 547000542..9f87cba03 100644 --- a/backend/go/vllm-cpp/video_test.go +++ b/backend/go/vllm-cpp/video_test.go @@ -282,6 +282,63 @@ var _ = Describe("GenerateVideo preconditions", func() { }) }) +var _ = Describe("buildLoraExtras", func() { + It("returns nil when no adapters are configured", func() { + keys, vals := buildLoraExtras(nil, nil, "", 0, "/models") + Expect(keys).To(BeNil()) + Expect(vals).To(BeNil()) + }) + + It("builds indexed lora_path/lora_strength pairs for one adapter", func() { + keys, vals := buildLoraExtras( + []string{"/abs/lora.safetensors"}, []float32{0.7}, "", 0, "/models") + Expect(keys).To(Equal([]string{"lora_path", "lora_strength"})) + Expect(vals).To(Equal([]string{"/abs/lora.safetensors", "0.7"})) + }) + + It("appends _N suffix from the second adapter onward", func() { + keys, vals := buildLoraExtras( + []string{"/a.safetensors", "/b.safetensors"}, + []float32{0.5, 0.3}, "", 0, "") + Expect(keys).To(Equal([]string{ + "lora_path", "lora_strength", + "lora_path_2", "lora_strength_2", + })) + Expect(vals).To(Equal([]string{ + "/a.safetensors", "0.5", + "/b.safetensors", "0.3", + })) + }) + + It("defaults strength to 1 when no scale is given", func() { + keys, vals := buildLoraExtras( + []string{"/a.safetensors"}, nil, "", 0, "") + Expect(keys).To(Equal([]string{"lora_path", "lora_strength"})) + Expect(vals).To(Equal([]string{"/a.safetensors", "1"})) + }) + + It("folds the singular lora_adapter/lora_scale in as the first adapter", func() { + keys, vals := buildLoraExtras( + []string{"/second.safetensors"}, []float32{0.2}, + "/first.safetensors", 0.8, "") + Expect(keys).To(Equal([]string{ + "lora_path", "lora_strength", + "lora_path_2", "lora_strength_2", + })) + Expect(vals).To(Equal([]string{ + "/first.safetensors", "0.8", + "/second.safetensors", "0.2", + })) + }) + + It("resolves relative adapter paths against the models directory", func() { + keys, vals := buildLoraExtras( + []string{"loras/style.safetensors"}, nil, "", 0, "/models") + Expect(keys).To(Equal([]string{"lora_path", "lora_strength"})) + Expect(vals[0]).To(Equal(filepath.Join("/models", "loras/style.safetensors"))) + }) +}) + // writePPM writes a valid P6 header of the given geometry. Only the header is // read by anything under test, so the pixel payload is left off. func writePPM(width, height int) string {