From 99e6705aa4f0cc2f2cd9f4ad358bba2f31d64687 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Sun, 20 Sep 2026 12:05:27 +0000 Subject: [PATCH] feat(gallery): add Hy-MT2 7B GGUF variants Offer the official Q4_K_M, Q6_K, and Q8_0 builds for translation. Pin the downloads and document installation and translation prompts. Assisted-by: Codex:gpt-6 --- docs/content/features/model-gallery.md | 20 +++++ gallery/index.yaml | 117 +++++++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index bc8e524c6..c197e5e64 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -34,6 +34,26 @@ The entries can return after LocalAI ships a compatible backend. See [the compatibility issue](https://github.com/mudler/LocalAI/issues/11681) and [upstream llama.cpp support](https://github.com/ggml-org/llama.cpp/pull/26467). +## Hy-MT2-7B translation + +Install Tencent's [Hy-MT2-7B](https://huggingface.co/tencent/Hy-MT2-7B) translation model with: + +```bash +local-ai models install hy-mt2-7b-q4 +``` + +The entry offers Q4_K_M, Q6_K, and Q8_0 GGUF builds for the `llama-cpp` backend. +LocalAI selects a variant according to available memory. To select Q4_K_M explicitly, use: + +```bash +local-ai models install hy-mt2-7b-q4 --variant hy-mt2-7b-q4 +``` + +Include the target language in the user message, for example: +`Translate the following text into Italian, without additional explanation: Hello, how are you?` +The configuration uses the model's embedded chat template and an 8,192-token context window. +Increase `context_size` for longer documents if memory permits; the model supports up to 262,144 tokens. + ## Useful Links and resources - [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) - here you can find a list of the most performing models on the Open LLM benchmark. Keep in mind models compatible with LocalAI must be quantized in the `gguf` format. diff --git a/gallery/index.yaml b/gallery/index.yaml index 04fa669b6..86185fed8 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -2563,6 +2563,123 @@ - filename: llama-cpp/mmproj/Huihui-Qwen3.8-27B-abliterated-bf16/mmproj-model-bf16.gguf uri: huggingface://huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF/mmproj-model-bf16.gguf sha256: c9a09064683620bea3d3bfed5d4462e1a97a7d2fff7e5045d6862a0a85eeb5b5 +- name: "hy-mt2-7b-q4" + variants: + - model: hy-mt2-7b-q6 + - model: hy-mt2-7b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/tencent/Hy-MT2-7B + - https://huggingface.co/tencent/Hy-MT2-7B-GGUF + description: | + Hy-MT2-7B is Tencent's 7B multilingual translation model. It supports + translation instructions across 33 languages, including terminology + control and style transfer. This Q4_K_M GGUF uses the embedded chat + template with an 8K context window. Include the target language in + your prompt. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - multilingual + - translation + overrides: + backend: llama-cpp + context_size: 8192 + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/hy-mt2-7b/Hy-MT2-7B-Q4_K_M.gguf + repeat_penalty: 1.05 + temperature: 0.7 + top_k: 20 + top_p: 0.6 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/hy-mt2-7b/Hy-MT2-7B-Q4_K_M.gguf + uri: https://huggingface.co/tencent/Hy-MT2-7B-GGUF/resolve/ab8472660ac61fac25f1af43fac2599d52a8a775/Hy-MT2-7B-Q4_K_M.gguf + sha256: 9f96256500f3fc1ab4d64336b58f52a949a95ad7516b0c229476eef782f9f77b +- name: "hy-mt2-7b-q6" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/tencent/Hy-MT2-7B + - https://huggingface.co/tencent/Hy-MT2-7B-GGUF + description: | + Hy-MT2-7B is Tencent's 7B multilingual translation model. It supports + translation instructions across 33 languages, including terminology + control and style transfer. This Q6_K GGUF uses the embedded chat + template with an 8K context window. Include the target language in + your prompt. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - multilingual + - translation + overrides: + backend: llama-cpp + context_size: 8192 + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/hy-mt2-7b/HY-MT2-7B-Q6_K.gguf + repeat_penalty: 1.05 + temperature: 0.7 + top_k: 20 + top_p: 0.6 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/hy-mt2-7b/HY-MT2-7B-Q6_K.gguf + uri: https://huggingface.co/tencent/Hy-MT2-7B-GGUF/resolve/ab8472660ac61fac25f1af43fac2599d52a8a775/HY-MT2-7B-Q6_K.gguf + sha256: 88ef0aba59952a4cfe4be36cb5baf797dbb370bc60e9dcbd7297036021e52831 +- name: "hy-mt2-7b-q8" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/tencent/Hy-MT2-7B + - https://huggingface.co/tencent/Hy-MT2-7B-GGUF + description: | + Hy-MT2-7B is Tencent's 7B multilingual translation model. It supports + translation instructions across 33 languages, including terminology + control and style transfer. This Q8_0 GGUF uses the embedded chat + template with an 8K context window. Include the target language in + your prompt. + license: "apache-2.0" + tags: + - llm + - gguf + - cpu + - gpu + - multilingual + - translation + overrides: + backend: llama-cpp + context_size: 8192 + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/hy-mt2-7b/HY-MT2-7B-Q8_0.gguf + repeat_penalty: 1.05 + temperature: 0.7 + top_k: 20 + top_p: 0.6 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/hy-mt2-7b/HY-MT2-7B-Q8_0.gguf + uri: https://huggingface.co/tencent/Hy-MT2-7B-GGUF/resolve/ab8472660ac61fac25f1af43fac2599d52a8a775/HY-MT2-7B-Q8_0.gguf + sha256: 58b3ad55dd6f6fa08c695cddc34fb5f8f708a844f78ae10508071914b0ed67c0 - &hy-mt2-1-8b name: "hy-mt2-1.8b-q4" variants: