diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 56dde2118..d4a184d4b 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -1112,3 +1112,20 @@ is still running is reported as queued until the installer picks it up: A job ID is queryable from the moment `/models/apply` returns it, so a `404`/`500` from this endpoint means the ID is genuinely unknown rather than merely waiting its turn. + +## Qwen3.8-27B iMatrix NVFP4 + +The `qwen3.8-27b-imatrix-nvfp4-mtp` variant uses cdiamond's mixed-precision +NVFP4 GGUF with llama.cpp. It includes the F16 vision projector and enables +MTP with one draft token per step. The default context is 32,768 tokens. + +Select this build explicitly with: + +```sh +local-ai models install qwen3.8-27b-q4 --variant qwen3.8-27b-imatrix-nvfp4-mtp +``` + +The model and projector total about 18.1 GB; inference requires additional +memory for context and working buffers. See the +[publisher's model card](https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF) +for the quantization recipe and hardware-specific measurements. diff --git a/gallery/index.yaml b/gallery/index.yaml index 981cc3563..d802532f5 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3,6 +3,68 @@ # written by scripts/models/cmd/modelcatalog and stays only so releases before # the cutover keep working. CI fails when it differs from the generated copy: # go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: qwen3.8-27b-imatrix-nvfp4-mtp + description: | + Qwen3.8-27B in cdiamond's iMatrix-calibrated mixed-precision NVFP4 + build. Selected attention and feed-forward tensors retain higher + precision. Includes the F16 vision projector and enables the embedded + MTP draft head with one draft token per step. + license: apache-2.0 + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + tags: + - llm + - gguf + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - nvfp4 + - mtp + - speculative + files: + - filename: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + sha256: 15b6bb8d0eb15cb6b081c562f8daea86330ec64174cb0f9e1655f3d467484007 + uri: https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF/resolve/ac343e8f44caef0896f79d372ecc07ef7ab34ec8/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + - filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-iMatrix-F16.gguf + sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e + uri: https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF/resolve/ac343e8f44caef0896f79d372ecc07ef7ab34ec8/mmproj-Qwen3.8-27B-F16.gguf + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-iMatrix-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:1 + - spec_p_min:0.75 + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + variants: [] - url: github:mudler/LocalAI/gallery/virtual.yaml@master name: adavar-3b-q4 description: | @@ -7088,6 +7150,7 @@ variants: - model: qwen3.8-27b-q4-mtp - model: qwen3.8-27b-nvfp4-mtp + - model: qwen3.8-27b-imatrix-nvfp4-mtp - model: qwen3.8-27b-ridge - model: qwen3.8-27b-gsq-rco-iq2-xs - model: qwen3.8-27b-gsq-rco-iq2-s @@ -7204,7 +7267,7 @@ known_usecases: - chat - vision - mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf options: - use_jinja:true - spec_type:draft-mtp diff --git a/gallery/models/index.yaml b/gallery/models/index.yaml index da124ab3a..f8ad06aab 100644 --- a/gallery/models/index.yaml +++ b/gallery/models/index.yaml @@ -4,6 +4,76 @@ # changes and never edit it by hand. CI fails when it is stale. A build from # master reads it as the default model source; a tagged build reads the gallery # until the release cutover. +- kind: model + id: qwen3.8-27b-imatrix-nvfp4-mtp + name: qwen3.8-27b-imatrix-nvfp4-mtp + version: 0.0.0-gallery.6dcacd1339cc + description: | + Qwen3.8-27B in cdiamond's iMatrix-calibrated mixed-precision NVFP4 + build. Selected attention and feed-forward tensors retain higher + precision. Includes the F16 vision projector and enables the embedded + MTP draft head with one draft token per step. + license: apache-2.0 + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF + tags: + - llm + - gguf + - gpu + - qwen + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + - nvfp4 + - mtp + - speculative + icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + sha256: 15b6bb8d0eb15cb6b081c562f8daea86330ec64174cb0f9e1655f3d467484007 + uri: https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF/resolve/ac343e8f44caef0896f79d372ecc07ef7ab34ec8/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + - filename: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-iMatrix-F16.gguf + sha256: cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e + uri: https://huggingface.co/cdiamond/Qwen3.8-27B-iMatrix-NVFP4-MTP-GGUF/resolve/ac343e8f44caef0896f79d372ecc07ef7ab34ec8/mmproj-Qwen3.8-27B-F16.gguf + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-iMatrix-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:1 + - spec_p_min:0.75 + parameters: + min_p: 0 + model: llama-cpp/models/qwen3.8-27b/Qwen3.8-27B-iMatrix-NVFP4-MTP.gguf + presence_penalty: 0 + repeat_penalty: 1 + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + variants: [] - kind: model id: adavar-3b-q4 name: adavar-3b-q4 @@ -8209,6 +8279,7 @@ variants: - model: qwen3.8-27b-q4-mtp - model: qwen3.8-27b-nvfp4-mtp + - model: qwen3.8-27b-imatrix-nvfp4-mtp - model: qwen3.8-27b-ridge - model: qwen3.8-27b-gsq-rco-iq2-xs - model: qwen3.8-27b-gsq-rco-iq2-s @@ -8341,7 +8412,7 @@ known_usecases: - chat - vision - mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-Q8_0.gguf + mmproj: llama-cpp/mmproj/qwen3.8-27b/mmproj-Qwen3.8-27B-BF16.gguf options: - use_jinja:true - spec_type:draft-mtp