From a8ee0d95ec75beab3cd6e655fca95aeb1218df05 Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Sun, 11 Oct 2026 00:26:54 +0200 Subject: [PATCH] feat(stablediffusion-ggml): add Qwen Image runtime schedules (#12619) * feat(stablediffusion-ggml): pass custom sigmas Turbo diffusion models require checkpoint-specific sigma schedules. Parse the model option and forward its values to stable-diffusion.cpp for each image request. Signed-off-by: Ettore Di Giacinto Assisted-by: nib * feat(gallery): add Qwen Image 2.1 runtimes Add the validated Anime FUSAL and AtomicChat Turbo Uncensored assemblies with pinned artifacts, exact runtime settings, licensing disclosures, and text-to-image limitations. Signed-off-by: Ettore Di Giacinto Assisted-by: nib --------- Signed-off-by: Ettore Di Giacinto Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- backend/go/stablediffusion-ggml/cpp/gosd.cpp | 5 + backend/go/stablediffusion-ggml/cpp/gosd.h | 1 + backend/go/stablediffusion-ggml/gosd.go | 46 +++++++++ backend/go/stablediffusion-ggml/gosd_test.go | 56 +++++++++-- backend/go/stablediffusion-ggml/main.go | 1 + gallery/index.yaml | 98 ++++++++++++++++++++ 6 files changed, 201 insertions(+), 6 deletions(-) diff --git a/backend/go/stablediffusion-ggml/cpp/gosd.cpp b/backend/go/stablediffusion-ggml/cpp/gosd.cpp index 74b2a0387..676163a8c 100644 --- a/backend/go/stablediffusion-ggml/cpp/gosd.cpp +++ b/backend/go/stablediffusion-ggml/cpp/gosd.cpp @@ -802,6 +802,11 @@ void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed) { params->seed = seed; } +void sd_img_gen_params_set_custom_sigmas(sd_img_gen_params_t *params, float *sigmas, int count) { + params->sample_params.custom_sigmas = sigmas; + params->sample_params.custom_sigmas_count = count; +} + int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count) { sd_image_t* results; diff --git a/backend/go/stablediffusion-ggml/cpp/gosd.h b/backend/go/stablediffusion-ggml/cpp/gosd.h index c6613d4ed..4b94b6009 100644 --- a/backend/go/stablediffusion-ggml/cpp/gosd.h +++ b/backend/go/stablediffusion-ggml/cpp/gosd.h @@ -15,6 +15,7 @@ sd_img_gen_params_t* sd_img_gen_params_new(void); void sd_img_gen_params_set_prompts(sd_img_gen_params_t *params, const char *prompt, const char *negative_prompt); void sd_img_gen_params_set_dimensions(sd_img_gen_params_t *params, int width, int height); void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed); +void sd_img_gen_params_set_custom_sigmas(sd_img_gen_params_t *params, float *sigmas, int count); int load_model(const char *model, char *model_path, char* options[], int threads, int diffusionModel); int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char* ref_images[], int ref_images_count); diff --git a/backend/go/stablediffusion-ggml/gosd.go b/backend/go/stablediffusion-ggml/gosd.go index 60f2efede..9970cefd5 100644 --- a/backend/go/stablediffusion-ggml/gosd.go +++ b/backend/go/stablediffusion-ggml/gosd.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "math" "os" "path/filepath" "runtime" @@ -19,6 +20,7 @@ type SDGGML struct { threads int sampleMethod string cfgScale float32 + customSigmas []float32 vaeTiling vaeTiling } @@ -36,6 +38,7 @@ var ( ImgGenParamsSetPrompts func(params uintptr, prompt string, negativePrompt string) ImgGenParamsSetDimensions func(params uintptr, width int, height int) ImgGenParamsSetSeed func(params uintptr, seed int64) + ImgGenParamsSetCustomSigmas func(params uintptr, sigmas uintptr, count int) ImgGenParamsGetVaeTilingParams func(params uintptr) uintptr VidGenParamsNew func() uintptr @@ -54,6 +57,40 @@ type vaeTiling struct { hasOverlap bool } +func parseCustomSigmas(options []string) ([]float32, error) { + for _, op := range options { + name, value, found := strings.Cut(op, ":") + if name != "custom_sigmas" { + continue + } + if !found { + return nil, fmt.Errorf("custom_sigmas requires a comma-separated value") + } + + values := strings.Split(value, ",") + if len(values) < 2 { + return nil, fmt.Errorf("custom_sigmas requires at least 2 values") + } + sigmas := make([]float32, len(values)) + for i, value := range values { + value = strings.TrimSpace(value) + if value == "" { + return nil, fmt.Errorf("custom_sigmas value %d is empty", i+1) + } + parsed, err := strconv.ParseFloat(value, 32) + if err != nil { + return nil, fmt.Errorf("custom_sigmas value %d %q is not a number", i+1, value) + } + if math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return nil, fmt.Errorf("custom_sigmas value %d must be finite", i+1) + } + sigmas[i] = float32(parsed) + } + return sigmas, nil + } + return nil, nil +} + func parseVAETiling(options []string) vaeTiling { var t vaeTiling for _, op := range options { @@ -158,6 +195,11 @@ func (sd *SDGGML) Load(opts *pb.ModelOptions) error { } sd.cfgScale = opts.CFGScale + sigmas, err := parseCustomSigmas(opts.Options) + if err != nil { + return fmt.Errorf("invalid model option: %w", err) + } + sd.customSigmas = sigmas // Read from the unfiltered list: none of the tiling options name a path, so // the resolution pass above neither rewrites nor drops them. sd.vaeTiling = parseVAETiling(opts.Options) @@ -209,6 +251,9 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error { ImgGenParamsSetPrompts(p, t, negative) ImgGenParamsSetDimensions(p, int(opts.Width), int(opts.Height)) ImgGenParamsSetSeed(p, int64(opts.Seed)) + if len(sd.customSigmas) > 0 { + ImgGenParamsSetCustomSigmas(p, uintptr(unsafe.Pointer(&sd.customSigmas[0])), len(sd.customSigmas)) + } // Tiling decodes the latent in overlapping tiles, so the VAE compute buffer // scales with the tile rather than with the image. That is the difference // between working and failing on any device that caps a single allocation @@ -227,6 +272,7 @@ func (sd *SDGGML) GenerateImage(opts *pb.GenerateImageRequest) error { } ret := GenImage(p, int(opts.Step), dst, sd.cfgScale, srcImage, strength, maskImage, refImages, refImagesCount) + runtime.KeepAlive(sd.customSigmas) runtime.KeepAlive(keepAlive) fmt.Fprintf(os.Stderr, "GenImage: %d\n", ret) if ret != 0 { diff --git a/backend/go/stablediffusion-ggml/gosd_test.go b/backend/go/stablediffusion-ggml/gosd_test.go index 3476700c6..beca7b3da 100644 --- a/backend/go/stablediffusion-ggml/gosd_test.go +++ b/backend/go/stablediffusion-ggml/gosd_test.go @@ -2,6 +2,7 @@ package main import ( "testing" + "unsafe" pb "github.com/mudler/LocalAI/pkg/grpc/proto" . "github.com/onsi/ginkgo/v2" @@ -13,6 +14,27 @@ func TestStableDiffusionGGML(t *testing.T) { RunSpecs(t, "stablediffusion-ggml backend test suite") } +var _ = DescribeTable("parseCustomSigmas", + func(options []string, want []float32, wantErr string) { + got, err := parseCustomSigmas(options) + if wantErr != "" { + Expect(err).To(MatchError(ContainSubstring(wantErr))) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(want)) + }, + Entry("valid schedule", []string{"custom_sigmas:1.0,0.5,0.0"}, []float32{1, 0.5, 0}, ""), + Entry("trims whitespace", []string{"custom_sigmas: 1.0, 0.5 , 0.0 "}, []float32{1, 0.5, 0}, ""), + Entry("absent", []string{"sampler:euler"}, nil, ""), + Entry("empty element", []string{"custom_sigmas:1.0,,0.0"}, nil, "value 2 is empty"), + Entry("non-numeric value", []string{"custom_sigmas:1.0,nope"}, nil, `value 2 "nope" is not a number`), + Entry("NaN", []string{"custom_sigmas:1.0,NaN"}, nil, "value 2 must be finite"), + Entry("positive infinity", []string{"custom_sigmas:1.0,+Inf"}, nil, "value 2 must be finite"), + Entry("negative infinity", []string{"custom_sigmas:1.0,-Inf"}, nil, "value 2 must be finite"), + Entry("one value", []string{"custom_sigmas:1.0"}, nil, "at least 2 values"), +) + var _ = DescribeTable("parseVAETiling enablement", func(options []string, want bool) { Expect(parseVAETiling(options).enabled).To(Equal(want)) @@ -76,12 +98,14 @@ var _ = DescribeTable("parseVAETiling target overlap", // available here; every one the code under test touches must be set or the // call panics on a nil func. type fakeSDLib struct { - tilingEnabled bool - tileSizeCalls int - tileSizeX int - tileSizeY int - overlapCalls int - targetOverlap float32 + tilingEnabled bool + tileSizeCalls int + tileSizeX int + tileSizeY int + overlapCalls int + targetOverlap float32 + customSigmasCalls int + customSigmas []float32 } // install points the bindings at the recorder and restores them afterwards, so @@ -91,6 +115,7 @@ func (f *fakeSDLib) install() { savedImgGenParamsSetPrompts := ImgGenParamsSetPrompts savedImgGenParamsSetDimensions := ImgGenParamsSetDimensions savedImgGenParamsSetSeed := ImgGenParamsSetSeed + savedImgGenParamsSetCustomSigmas := ImgGenParamsSetCustomSigmas savedImgGenParamsGetVaeTilingParams := ImgGenParamsGetVaeTilingParams savedTilingParamsSetEnabled := TilingParamsSetEnabled savedTilingParamsSetTileSizes := TilingParamsSetTileSizes @@ -103,6 +128,7 @@ func (f *fakeSDLib) install() { ImgGenParamsSetPrompts = savedImgGenParamsSetPrompts ImgGenParamsSetDimensions = savedImgGenParamsSetDimensions ImgGenParamsSetSeed = savedImgGenParamsSetSeed + ImgGenParamsSetCustomSigmas = savedImgGenParamsSetCustomSigmas ImgGenParamsGetVaeTilingParams = savedImgGenParamsGetVaeTilingParams TilingParamsSetEnabled = savedTilingParamsSetEnabled TilingParamsSetTileSizes = savedTilingParamsSetTileSizes @@ -115,6 +141,10 @@ func (f *fakeSDLib) install() { ImgGenParamsSetPrompts = func(uintptr, string, string) {} ImgGenParamsSetDimensions = func(uintptr, int, int) {} ImgGenParamsSetSeed = func(uintptr, int64) {} + ImgGenParamsSetCustomSigmas = func(_ uintptr, sigmas uintptr, count int) { + f.customSigmasCalls++ + f.customSigmas = append([]float32(nil), unsafe.Slice((*float32)(unsafe.Pointer(sigmas)), count)...) + } ImgGenParamsGetVaeTilingParams = func(uintptr) uintptr { return 2 } TilingParamsSetEnabled = func(_ uintptr, enabled bool) { f.tilingEnabled = enabled } TilingParamsSetTileSizes = func(_ uintptr, x, y int) { @@ -145,6 +175,20 @@ var _ = Describe("GenerateImage VAE tiling", func() { Expect(sd.GenerateImage(&pb.GenerateImageRequest{Width: 1024, Height: 1024})).To(Succeed()) } + It("forwards custom sigmas to image generation", func() { + generate([]string{"custom_sigmas:1.0,0.5,0.0"}) + + Expect(fake.customSigmasCalls).To(Equal(1)) + Expect(fake.customSigmas).To(Equal([]float32{1, 0.5, 0})) + }) + + It("does not set custom sigmas when the option is absent", func() { + generate([]string{"sampler:euler"}) + + Expect(fake.customSigmasCalls).To(BeZero()) + Expect(fake.customSigmas).To(BeNil()) + }) + It("enables tiling when the model asks for it", func() { generate([]string{"vae_tiling:true"}) diff --git a/backend/go/stablediffusion-ggml/main.go b/backend/go/stablediffusion-ggml/main.go index b509c6a2b..2f2badca8 100644 --- a/backend/go/stablediffusion-ggml/main.go +++ b/backend/go/stablediffusion-ggml/main.go @@ -47,6 +47,7 @@ func main() { {&ImgGenParamsSetPrompts, "sd_img_gen_params_set_prompts"}, {&ImgGenParamsSetDimensions, "sd_img_gen_params_set_dimensions"}, {&ImgGenParamsSetSeed, "sd_img_gen_params_set_seed"}, + {&ImgGenParamsSetCustomSigmas, "sd_img_gen_params_set_custom_sigmas"}, {&ImgGenParamsGetVaeTilingParams, "sd_img_gen_params_get_vae_tiling_params"}, {&VidGenParamsNew, "sd_vid_gen_params_new"}, diff --git a/gallery/index.yaml b/gallery/index.yaml index 19447185a..400149a0d 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -47978,6 +47978,104 @@ - filename: qwen_image_2.1_vae_bf16.safetensors sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9 uri: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/resolve/main/vae/qwen_image_2.1_vae_bf16.safetensors +- name: qwen-image-2.1-anime-fusal-gguf + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/Qwen/Qwen-Image-2.1 + - https://huggingface.co/leejet/Qwen-Image-2.1-GGUF + - https://huggingface.co/DreamFast/Qwen3-VL-8B-Heretic-1.3.0 + - https://huggingface.co/MicksHF/Qwen-Image-2.1-Anime-Fusal + - https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF + description: | + Qwen-Image 2.1 Anime FUSAL runtime assembly with a Q5_0 denoiser, Heretic Q4_K_M text encoder, BF16 VAE, and separate rank-80 FUSAL adapter. Downloads total about 10.4 GiB; allow at least that much combined RAM/VRAM plus runtime overhead. + + This assembly has mixed licensing: the Qwen-derived denoiser, VAE, and FUSAL adapter are under the Qwen Research License and are limited to non-commercial research or evaluation unless separately authorized; the Heretic encoder is Apache-2.0. Review the repository notices for NSFW content and third-party-rights responsibilities. + + Use 30 steps, Euler sampling, and CFG 1.0. The adapter is applied at runtime, so prompts must use `fusal style. [TASK: TEXT_TO_IMAGE] DESCRIPTION`, replacing DESCRIPTION with the image description. This practical encoder omits vision weights, so this entry is text-to-image only and does not support reference-image editing. + license: other + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_logo.png + tags: + - qwen + - qwen-image + - text-to-image + - anime + - gguf + - quantized + - diffusion + - lora + last_checked: "2026-10-10" + overrides: + backend: stablediffusion-ggml + step: 30 + cfg_scale: 1 + options: + - diffusion_model + - llm_path:qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf + - vae_path:qwen_image_2.1_vae_bf16.safetensors + - sampler:euler + - offload_params_to_cpu:true + parameters: + model: qwen_image_2.1-Q5_0.gguf + files: + - filename: qwen_image_2.1-Q5_0.gguf + sha256: 8dfeb1ee091a5d7c7d8191f69723254f031de8fe67ec219a21bf7df57590ab43 + uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen_image_2.1-Q5_0.gguf + - filename: qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf + sha256: fe3510473630bfb8092f94900eec3d4847f7658caaaf577981205af29b438292 + uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen3-vl-8b-heretic-1.3.0-Q4_K_M.gguf + - filename: qwen_image_2.1_vae_bf16.safetensors + sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9 + uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/qwen_image_2.1_vae_bf16.safetensors + - filename: fusal-qwen-image-2.1-stack-exact-r80.safetensors + sha256: 397e4e6f9d69942f901b0d7a7fcecbb25cc3002c1534e90b90380b0ce95b0836 + uri: https://huggingface.co/LocalAI-io/Qwen-Image-2.1-Anime-Fusal-GGUF/resolve/b183680b3f0e705017781d93b050d00305c2a482/fusal-qwen-image-2.1-stack-exact-r80.safetensors +- name: qwen-image-2.1-turbo-uncensored-gguf + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/Qwen/Qwen-Image-2.1 + - https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-GGUF + - https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-Uncensored-GGUF + - https://huggingface.co/Comfy-Org/Qwen-Image-2.1 + description: | + Qwen-Image 2.1 Turbo text-to-image assembly with AtomicChat's AD-Q5_K denoiser, Q8_0 abliterated text encoder, and the Qwen Image 2.1 BF16 VAE. It uses the Turbo runtime's exact eight-step sigma schedule with Euler sampling and CFG 1.0. + + The denoiser and VAE are governed by the Qwen Research License; the separately modified encoder is Apache-2.0. The complete image runtime is therefore not Apache-2.0. AtomicChat reports that changing the encoder's refusal direction produced no measured category-outcome change in its image tests, so this entry does not promise broader image capability. + + This bundle has no vision projector and supports text-to-image only, not reference-image editing. + license: other + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/qwen_image_logo.png + tags: + - qwen + - qwen-image + - text-to-image + - uncensored + - gguf + - quantized + - diffusion + last_checked: "2026-10-10" + overrides: + backend: stablediffusion-ggml + step: 8 + cfg_scale: 1 + options: + - diffusion_model + - llm_path:Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf + - vae_path:qwen_image_2.1_vae_bf16.safetensors + - sampler:euler + - custom_sigmas:1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0 + - offload_params_to_cpu:true + parameters: + model: Qwen-Image-2.1-Turbo-AD-Q5_K.gguf + files: + - filename: Qwen-Image-2.1-Turbo-AD-Q5_K.gguf + sha256: c56db41508c4c3c0d5782e40e4f153c38a1154ac9466c1c5c50a6e4bd09f6a69 + uri: https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-GGUF/resolve/bb25d06bc74119c12207243d68917951e6d9c232/Qwen-Image-2.1-Turbo-AD-Q5_K.gguf + - filename: Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf + sha256: f8f9f6276b726db5291f4a5343089612cb5fd08aa81e8f0791dc66d2c03d07ca + uri: https://huggingface.co/AtomicChat/Qwen-Image-2.1-Turbo-Uncensored-GGUF/resolve/e3b01e9bfacc373148bdb93824496e0c72643250/Qwen-Image-2.1-Turbo-Abliterated-Uncensored-Q8_0.gguf + - filename: qwen_image_2.1_vae_bf16.safetensors + sha256: bb21f7473051e1ac368515dd3f2e15cd44d7a11748ee8823e1ddca3e4876b7c9 + uri: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/resolve/df94239739eef7205973a145ffa6f441f7b64e84/vae/qwen_image_2.1_vae_bf16.safetensors - name: whisper-1 url: github:mudler/LocalAI/gallery/whisper-base.yaml@master urls: