From fbed25f1f7b359e40f979c6fe68c07d683bf7744 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
 <306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Mon, 14 Sep 2026 12:05:11 +0000
Subject: [PATCH] feat(gallery): add Qwen3.8 9B Distill Q5 and Q6

Fill the quantization gap between the existing Q4 and Q8 builds.
Pin the new downloads to a Hugging Face revision and retain the
existing llama.cpp chat configuration. Document explicit selection.

Assisted-by: Codex:GPT-6
---
 docs/content/features/model-gallery.md | 15 ++++++
 gallery/index.yaml                     | 66 +++++++++++++++++++++++++-
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md
index f0088f583..80befaccf 100644
--- a/docs/content/features/model-gallery.md
+++ b/docs/content/features/model-gallery.md
@@ -90,6 +90,21 @@ temperature 1, top_k 64, and top_p 0.95. They are separate from the
 existing QAT builds. See the [source model](https://huggingface.co/google/gemma-4-12B-it)
 and [GGUF files](https://huggingface.co/unsloth/gemma-4-12b-it-GGUF).
 
+### Qwen3.8 9B Distill quantizations
+
+The text-only Qwen3.8 9B Distill entry offers Q4_K_M, Q5_K_M, Q6_K, and
+Q8_0 GGUF builds for llama.cpp. Select Q5 or Q6 explicitly with:
+
+```bash
+local-ai models install qwen3.8-9b-distill-q4 --variant qwen3.8-9b-distill-q5
+local-ai models install qwen3.8-9b-distill-q4 --variant qwen3.8-9b-distill-q6
+```
+
+All four builds use the embedded chat template, a 32,768-token context,
+and the same sampling defaults. See the
+[GGUF repository](https://huggingface.co/empero-ai/Qwen3.8-9B-Distill-GGUF)
+for the published quantizations.
+
 ## Add other galleries
 
 You can add other galleries by:
diff --git a/gallery/index.yaml b/gallery/index.yaml
index 104827cc3..f184daba2 100644
--- a/gallery/index.yaml
+++ b/gallery/index.yaml
@@ -5902,6 +5902,8 @@
 - &qwen3-8-9b-distill
   name: "qwen3.8-9b-distill-q4"
   variants:
+    - model: qwen3.8-9b-distill-q5
+    - model: qwen3.8-9b-distill-q6
     - model: qwen3.8-9b-distill-q8
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:
@@ -5912,7 +5914,7 @@
     distilled from Qwen3.8 2.4T A95B reasoning traces. It targets mathematics,
     coding, instruction following, and function calling with a 262K native
     context window. This entry uses the balanced Q4_K_M GGUF quantization; the
-    Q8_0 variant offers higher fidelity.
+    Q5_K_M, Q6_K, and Q8_0 variants offer higher fidelity.
   license: apache-2.0
   icon: https://qianwen-res.oss-cn-beijing.aliyuncs.com/logo_qwen.jpg
   tags:
@@ -5978,6 +5980,68 @@
     - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q8_0.gguf
       uri: huggingface://empero-ai/Qwen3.8-9B-Distill-GGUF/Qwen3.8-9B-Q8_0.gguf
       sha256: 79ca5d342a07922f2bbf38c8d892a79a3c8620c65feaf4b1c66b7830ae724db8
+- !!merge <<: *qwen3-8-9b-distill
+  name: "qwen3.8-9b-distill-q5"
+  variants: []
+  last_checked: "2026-09-14"
+  description: |
+    Qwen3.8 9B Distill in the higher-fidelity Q5_K_M GGUF format. This text-only
+    Qwen3.5 9B fine-tune targets reasoning, coding, instruction following, and
+    function calling with a 262K native context window.
+  overrides:
+    backend: llama-cpp
+    context_size: 32768
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    options:
+      - use_jinja:true
+      - reasoning_format:deepseek
+    parameters:
+      model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q5_K_M.gguf
+      temperature: 0.6
+      top_k: 20
+      top_p: 0.95
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q5_K_M.gguf
+      uri: https://huggingface.co/empero-ai/Qwen3.8-9B-Distill-GGUF/resolve/760121cd70bb4c36b2b5ec58eb765e0df5987efe/Qwen3.8-9B-Q5_K_M.gguf
+      sha256: c6667345d4e45d8cddca3c8e997a483a4f9ae04ee9402273c24e959dee7173dc
+- !!merge <<: *qwen3-8-9b-distill
+  name: "qwen3.8-9b-distill-q6"
+  variants: []
+  last_checked: "2026-09-14"
+  description: |
+    Qwen3.8 9B Distill in the higher-fidelity Q6_K GGUF format. This text-only
+    Qwen3.5 9B fine-tune targets reasoning, coding, instruction following, and
+    function calling with a 262K native context window.
+  overrides:
+    backend: llama-cpp
+    context_size: 32768
+    function:
+      automatic_tool_parsing_fallback: true
+      grammar:
+        disable: true
+    known_usecases:
+      - chat
+    options:
+      - use_jinja:true
+      - reasoning_format:deepseek
+    parameters:
+      model: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q6_K.gguf
+      temperature: 0.6
+      top_k: 20
+      top_p: 0.95
+    template:
+      use_tokenizer_template: true
+  files:
+    - filename: llama-cpp/models/qwen3.8-9b-distill/Qwen3.8-9B-Q6_K.gguf
+      uri: https://huggingface.co/empero-ai/Qwen3.8-9B-Distill-GGUF/resolve/760121cd70bb4c36b2b5ec58eb765e0df5987efe/Qwen3.8-9B-Q6_K.gguf
+      sha256: 0f1271373f899912bfe4ea76299af7dd83722d98ea421b0827501c3a2c6da22b
 - name: "btl-4-compact"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls: