feat(stablediffusion-ggml): add Qwen Image runtime schedules (#12619)

* feat(stablediffusion-ggml): pass custom sigmas

Turbo diffusion models require checkpoint-specific sigma schedules. Parse the model option and forward its values to stable-diffusion.cpp for each image request.

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
Assisted-by: nib <noreply@localai.io>

* feat(gallery): add Qwen Image 2.1 runtimes

Add the validated Anime FUSAL and AtomicChat Turbo Uncensored assemblies with pinned artifacts, exact runtime settings, licensing disclosures, and text-to-image limitations.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: nib <noreply@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -802,6 +802,11 @@ void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed) {
     params->seed = seed;
 }
 
+void sd_img_gen_params_set_custom_sigmas(sd_img_gen_params_t *params, float *sigmas, int count) {
+    params->sample_params.custom_sigmas = sigmas;
+    params->sample_params.custom_sigmas_count = count;
+}
+
 int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count) {
 
     sd_image_t* results;

backend/go/stablediffusion-ggml/cpp/gosd.h

@@ -15,6 +15,7 @@ sd_img_gen_params_t* sd_img_gen_params_new(void);
 void sd_img_gen_params_set_prompts(sd_img_gen_params_t *params, const char *prompt, const char *negative_prompt);
 void sd_img_gen_params_set_dimensions(sd_img_gen_params_t *params, int width, int height);
 void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed);
+void sd_img_gen_params_set_custom_sigmas(sd_img_gen_params_t *params, float *sigmas, int count);
 
 int load_model(const char *model, char *model_path, char* options[], int threads, int diffusionModel);
 int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count);

backend/go/stablediffusion-ggml/gosd.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -19,6 +20,7 @@ type SDGGML struct {
 	threads      int
 	sampleMethod string
 	cfgScale     float32
+	customSigmas []float32
 	vaeTiling    vaeTiling
 }
 
@@ -36,6 +38,7 @@ var (
 	ImgGenParamsSetPrompts         func(params uintptr, prompt string, negativePrompt string)
 	ImgGenParamsSetDimensions      func(params uintptr, width int, height int)
 	ImgGenParamsSetSeed            func(params uintptr, seed int64)
+	ImgGenParamsSetCustomSigmas    func(params uintptr, sigmas uintptr, count int)
 	ImgGenParamsGetVaeTilingParams func(params uintptr) uintptr
 
 	VidGenParamsNew            func() uintptr
@@ -54,6 +57,40 @@ type vaeTiling struct {
 	hasOverlap    bool
 }
 
+func parseCustomSigmas(options []string) ([]float32, error) {
+	for _, op := range options {
+		name, value, found := strings.Cut(op, ":")
+		if name != "custom_sigmas" {
+			continue
+		}
+		if !found {
+			return nil, fmt.Errorf("custom_sigmas requires a comma-separated value")
+		}
+
+		values := strings.Split(value, ",")
+		if len(values) < 2 {
+			return nil, fmt.Errorf("custom_sigmas requires at least 2 values")
+		}
+		sigmas := make([]float32, len(values))
+		for i, value := range values {
+			value = strings.TrimSpace(value)
+			if value == "" {
+				return nil, fmt.Errorf("custom_sigmas value %d is empty", i+1)
+			}
+			parsed, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				return nil, fmt.Errorf("custom_sigmas value %d %q is not a number", i+1, value)
+			}
+			if math.IsNaN(parsed) || math.IsInf(parsed, 0) {
+				return nil, fmt.Errorf("custom_sigmas value %d must be finite", i+1)
+			}
+			sigmas[i] = float32(parsed)
+		}
+		return sigmas, nil
+	}
+	return nil, nil
+}
+
 func parseVAETiling(options []string) vaeTiling {
 	var t vaeTiling
 	for _, op := range options {
@@ -158,6 +195,11 @@ func (sd *SDGGML) Load(opts *pb.ModelOptions) error {
 	}
 
 	sd.cfgScale = opts.CFGScale
+	sigmas, err := parseCustomSigmas(opts.Options)
+	if err != nil {
+		return fmt.Errorf("invalid model option: %w", err)
+	}
+	sd.customSigmas = sigmas
 	// Read from the unfiltered list: none of the tiling options name a path, so
 	// the resolution pass above neither rewrites nor drops them.
 	sd.vaeTiling = parseVAETiling(opts.Options)
@@ -209,6 +251,9 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error {
 	ImgGenParamsSetPrompts(p, t, negative)
 	ImgGenParamsSetDimensions(p, int(opts.Width), int(opts.Height))
 	ImgGenParamsSetSeed(p, int64(opts.Seed))
+	if len(sd.customSigmas) > 0 {
+		ImgGenParamsSetCustomSigmas(p, uintptr(unsafe.Pointer(&sd.customSigmas[0])), len(sd.customSigmas))
+	}
 	// Tiling decodes the latent in overlapping tiles, so the VAE compute buffer
 	// scales with the tile rather than with the image. That is the difference
 	// between working and failing on any device that caps a single allocation
@@ -227,6 +272,7 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error {
 	}
 
 	ret := GenImage(p, int(opts.Step), dst, sd.cfgScale, srcImage, strength, maskImage, refImages, refImagesCount)
+	runtime.KeepAlive(sd.customSigmas)
 	runtime.KeepAlive(keepAlive)
 	fmt.Fprintf(os.Stderr, "GenImage: %d\n", ret)
 	if ret != 0 {

backend/go/stablediffusion-ggml/gosd_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"testing"
+	"unsafe"
 
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	. "github.com/onsi/ginkgo/v2"
@@ -13,6 +14,27 @@ func TestStableDiffusionGGML(t *testing.T) {
 	RunSpecs(t, "stablediffusion-ggml backend test suite")
 }
 
+var _ = DescribeTable("parseCustomSigmas",
+	func(options []string, want []float32, wantErr string) {
+		got, err := parseCustomSigmas(options)
+		if wantErr != "" {
+			Expect(err).To(MatchError(ContainSubstring(wantErr)))
+			return
+		}
+		Expect(err).NotTo(HaveOccurred())
+		Expect(got).To(Equal(want))
+	},
+	Entry("valid schedule", []string{"custom_sigmas:1.0,0.5,0.0"}, []float32{1, 0.5, 0}, ""),
+	Entry("trims whitespace", []string{"custom_sigmas: 1.0, 0.5 , 0.0 "}, []float32{1, 0.5, 0}, ""),
+	Entry("absent", []string{"sampler:euler"}, nil, ""),
+	Entry("empty element", []string{"custom_sigmas:1.0,,0.0"}, nil, "value 2 is empty"),
+	Entry("non-numeric value", []string{"custom_sigmas:1.0,nope"}, nil, `value 2 "nope" is not a number`),
+	Entry("NaN", []string{"custom_sigmas:1.0,NaN"}, nil, "value 2 must be finite"),
+	Entry("positive infinity", []string{"custom_sigmas:1.0,+Inf"}, nil, "value 2 must be finite"),
+	Entry("negative infinity", []string{"custom_sigmas:1.0,-Inf"}, nil, "value 2 must be finite"),
+	Entry("one value", []string{"custom_sigmas:1.0"}, nil, "at least 2 values"),
+)
+
 var _ = DescribeTable("parseVAETiling enablement",
 	func(options []string, want bool) {
 		Expect(parseVAETiling(options).enabled).To(Equal(want))
@@ -76,12 +98,14 @@ var _ = DescribeTable("parseVAETiling target overlap",
 // available here; every one the code under test touches must be set or the
 // call panics on a nil func.
 type fakeSDLib struct {
-	tilingEnabled bool
-	tileSizeCalls int
-	tileSizeX     int
-	tileSizeY     int
-	overlapCalls  int
-	targetOverlap float32
+	tilingEnabled     bool
+	tileSizeCalls     int
+	tileSizeX         int
+	tileSizeY         int
+	overlapCalls      int
+	targetOverlap     float32
+	customSigmasCalls int
+	customSigmas      []float32
 }
 
 // install points the bindings at the recorder and restores them afterwards, so
@@ -91,6 +115,7 @@ func (f *fakeSDLib) install() {
 	savedImgGenParamsSetPrompts := ImgGenParamsSetPrompts
 	savedImgGenParamsSetDimensions := ImgGenParamsSetDimensions
 	savedImgGenParamsSetSeed := ImgGenParamsSetSeed
+	savedImgGenParamsSetCustomSigmas := ImgGenParamsSetCustomSigmas
 	savedImgGenParamsGetVaeTilingParams := ImgGenParamsGetVaeTilingParams
 	savedTilingParamsSetEnabled := TilingParamsSetEnabled
 	savedTilingParamsSetTileSizes := TilingParamsSetTileSizes
@@ -103,6 +128,7 @@ func (f *fakeSDLib) install() {
 		ImgGenParamsSetPrompts = savedImgGenParamsSetPrompts
 		ImgGenParamsSetDimensions = savedImgGenParamsSetDimensions
 		ImgGenParamsSetSeed = savedImgGenParamsSetSeed
+		ImgGenParamsSetCustomSigmas = savedImgGenParamsSetCustomSigmas
 		ImgGenParamsGetVaeTilingParams = savedImgGenParamsGetVaeTilingParams
 		TilingParamsSetEnabled = savedTilingParamsSetEnabled
 		TilingParamsSetTileSizes = savedTilingParamsSetTileSizes
@@ -115,6 +141,10 @@ func (f *fakeSDLib) install() {
 	ImgGenParamsSetPrompts = func(uintptr, string, string) {}
 	ImgGenParamsSetDimensions = func(uintptr, int, int) {}
 	ImgGenParamsSetSeed = func(uintptr, int64) {}
+	ImgGenParamsSetCustomSigmas = func(_ uintptr, sigmas uintptr, count int) {
+		f.customSigmasCalls++
+		f.customSigmas = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(sigmas)), count)...)
+	}
 	ImgGenParamsGetVaeTilingParams = func(uintptr) uintptr { return 2 }
 	TilingParamsSetEnabled = func(_ uintptr, enabled bool) { f.tilingEnabled = enabled }
 	TilingParamsSetTileSizes = func(_ uintptr, x, y int) {
@@ -145,6 +175,20 @@ var _ = Describe("GenerateImage VAE tiling", func() {
 		Expect(sd.GenerateImage(&pb.GenerateImageRequest{Width: 1024, Height: 1024})).To(Succeed())
 	}
 
+	It("forwards custom sigmas to image generation", func() {
+		generate([]string{"custom_sigmas:1.0,0.5,0.0"})
+
+		Expect(fake.customSigmasCalls).To(Equal(1))
+		Expect(fake.customSigmas).To(Equal([]float32{1, 0.5, 0}))
+	})
+
+	It("does not set custom sigmas when the option is absent", func() {
+		generate([]string{"sampler:euler"})
+
+		Expect(fake.customSigmasCalls).To(BeZero())
+		Expect(fake.customSigmas).To(BeNil())
+	})
+
 	It("enables tiling when the model asks for it", func() {
 		generate([]string{"vae_tiling:true"})
 

backend/go/stablediffusion-ggml/main.go

@@ -47,6 +47,7 @@ func main() {
 		{&ImgGenParamsSetPrompts, "sd_img_gen_params_set_prompts"},
 		{&ImgGenParamsSetDimensions, "sd_img_gen_params_set_dimensions"},
 		{&ImgGenParamsSetSeed, "sd_img_gen_params_set_seed"},
+		{&ImgGenParamsSetCustomSigmas, "sd_img_gen_params_set_custom_sigmas"},
 		{&ImgGenParamsGetVaeTilingParams, "sd_img_gen_params_get_vae_tiling_params"},
 
 		{&VidGenParamsNew, "sd_vid_gen_params_new"},

gallery/index.yaml

@@ -47978,6 +47978,104 @@
     - filename: qwen_image_2.1_vae_bf16.safetensors
       sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9
       uri: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/resolve/main/vae/qwen_image_2.1_vae_bf16.safetensors
+- name: qwen-image-2.1-anime-fusal-gguf
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Qwen/Qwen-Image-2.1
+    - https://huggingface.co/leejet/Qwen-Image-2.1-GGUF
+    - https://huggingface.co/DreamFast/Qwen3-VL-8B-Heretic-1.3.0
+    - https://huggingface.co/MicksHF/Qwen-Image-2.1-Anime-Fusal
+    - https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF
+  description: |
+    Qwen-Image 2.1 Anime FUSAL runtime assembly with a Q5_0 denoiser, Heretic Q4_K_M text encoder, BF16 VAE, and separate rank-80 FUSAL adapter. Downloads total about 10.4 GiB; allow at least that much combined RAM/VRAM plus runtime overhead.
+
+    This assembly has mixed licensing: the Qwen-derived denoiser, VAE, and FUSAL adapter are under the Qwen Research License and are limited to non-commercial research or evaluation unless separately authorized; the Heretic encoder is Apache-2.0. Review the repository notices for NSFW content and third-party-rights responsibilities.
+
+    Use 30 steps, Euler sampling, and CFG 1.0. The adapter is applied at runtime, so prompts must use `fusal style. [TASK: TEXT_TO_IMAGE] DESCRIPTION<lora:fusal-qwen-image-2.1-stack-exact-r80:1.0>`, replacing DESCRIPTION with the image description. This practical encoder omits vision weights, so this entry is text-to-image only and does not support reference-image editing.
+  license: other
+  icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_logo.png
+  tags:
+    - qwen
+    - qwen-image
+    - text-to-image
+    - anime
+    - gguf
+    - quantized
+    - diffusion
+    - lora
+  last_checked: "2026-10-10"
+  overrides:
+    backend: stablediffusion-ggml
+    step: 30
+    cfg_scale: 1
+    options:
+      - diffusion_model
+      - llm_path:qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf
+      - vae_path:qwen_image_2.1_vae_bf16.safetensors
+      - sampler:euler
+      - offload_params_to_cpu:true
+    parameters:
+      model: qwen_image_2.1-Q5_0.gguf
+  files:
+    - filename: qwen_image_2.1-Q5_0.gguf
+      sha256: 8dfeb1ee091a5d7c7d8191f69723254f031de8fe67ec219a21bf7df57590ab43
+      uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen_image_2.1-Q5_0.gguf
+    - filename: qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf
+      sha256: fe3510473630bfb8092f94900eec3d4847f7658caaaf577981205af29b438292
+      uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf
+    - filename: qwen_image_2.1_vae_bf16.safetensors
+      sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9
+      uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen_image_2.1_vae_bf16.safetensors
+    - filename: fusal-qwen-image-2.1-stack-exact-r80.safetensors
+      sha256: 397e4e6f9d69942f901b0d7a7fcecbb25cc3002c1534e90b90380b0ce95b0836
+      uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/fusal-qwen-image-2.1-stack-exact-r80.safetensors
+- name: qwen-image-2.1-turbo-uncensored-gguf
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/Qwen/Qwen-Image-2.1
+    - https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-GGUF
+    - https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-Uncensored-GGUF
+    - https://huggingface.co/Comfy-Org/Qwen-Image-2.1
+  description: |
+    Qwen-Image 2.1 Turbo text-to-image assembly with AtomicChat's AD-Q5_K denoiser, Q8_0 abliterated text encoder, and the Qwen Image 2.1 BF16 VAE. It uses the Turbo runtime's exact eight-step sigma schedule with Euler sampling and CFG 1.0.
+
+    The denoiser and VAE are governed by the Qwen Research License; the separately modified encoder is Apache-2.0. The complete image runtime is therefore not Apache-2.0. AtomicChat reports that changing the encoder's refusal direction produced no measured category-outcome change in its image tests, so this entry does not promise broader image capability.
+
+    This bundle has no vision projector and supports text-to-image only, not reference-image editing.
+  license: other
+  icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_logo.png
+  tags:
+    - qwen
+    - qwen-image
+    - text-to-image
+    - uncensored
+    - gguf
+    - quantized
+    - diffusion
+  last_checked: "2026-10-10"
+  overrides:
+    backend: stablediffusion-ggml
+    step: 8
+    cfg_scale: 1
+    options:
+      - diffusion_model
+      - llm_path:Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf
+      - vae_path:qwen_image_2.1_vae_bf16.safetensors
+      - sampler:euler
+      - custom_sigmas:1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0
+      - offload_params_to_cpu:true
+    parameters:
+      model: Qwen-Image-2.1-Turbo-AD-Q5_K.gguf
+  files:
+    - filename: Qwen-Image-2.1-Turbo-AD-Q5_K.gguf
+      sha256: c56db41508c4c3c0d5782e40e4f153c38a1154ac9466c1c5c50a6e4bd09f6a69
+      uri: https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-GGUF/resolve/bb25d06bc74119c12207243d68917951e6d9c232/Qwen-Image-2.1-Turbo-AD-Q5_K.gguf
+    - filename: Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf
+      sha256: f8f9f6276b726db5291f4a5343089612cb5fd08aa81e8f0791dc66d2c03d07ca
+      uri: https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-Uncensored-GGUF/resolve/e3b01e9bfacc373148bdb93824496e0c72643250/Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf
+    - filename: qwen_image_2.1_vae_bf16.safetensors
+      sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9
+      uri: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/resolve/df94239739eef7205973a145ffa6f441f7b64e84/vae/qwen_image_2.1_vae_bf16.safetensors
 - name: whisper-1
   url: github:mudler/LocalAI/gallery/whisper-base.yaml@master
   urls: