diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 56dde2118..912e3e31d 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -181,6 +181,9 @@ nothing is deleted. is a text-only reasoning model with 24B total parameters and 2.45B active parameters. The gallery provides the official Q4_K_M, Q5_K_M, Q6_K, and Q8_0 GGUF builds through the `llama-cpp` backend. +[Unsloth dynamic quantizations](https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF) +are also available as `lfm2.5-24b-a2b-ud-q4` (UD-Q4_K_XL) and +`lfm2.5-24b-a2b-ud-q6` (UD-Q6_K_XL) in the same variant group. Install the model with automatic variant selection: diff --git a/gallery/index.yaml b/gallery/index.yaml index 981cc3563..44585bbe5 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3,6 +3,86 @@ # written by scripts/models/cmd/modelcatalog and stays only so releases before # the cutover keep working. CI fails when it differs from the generated copy: # go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: lfm2.5-24b-a2b-ud-q4 + description: | + LiquidAI LFM2.5-24B-A2B is a text-only reasoning model with 24B total + parameters and 2.45B active parameters for multilingual chat and tool use. + This entry uses Unsloth's dynamic UD-Q4_K_XL GGUF quantization. + license: other + urls: + - https://huggingface.co/LiquidAI/LFM2.5-24B-A2B + - https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF + icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + files: + - filename: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + sha256: a235d23638cc5549cb1ff61da7522c8a7e97f056f27093462a1e56f35aba91d8 + uri: https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF/resolve/faa77113e93335b63423e9e8d33395f86eaf8946/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + repeat_penalty: 1.05 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: lfm2.5-24b-a2b-ud-q6 + description: | + LiquidAI LFM2.5-24B-A2B is a text-only reasoning model with 24B total + parameters and 2.45B active parameters for multilingual chat and tool use. + This entry uses Unsloth's dynamic UD-Q6_K_XL GGUF quantization. + license: other + urls: + - https://huggingface.co/LiquidAI/LFM2.5-24B-A2B + - https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF + icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + files: + - filename: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + sha256: abcb7e21e006eb4c826a2b75fb91c0804db7b20f908a61beeebdbdd07c88f41d + uri: https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF/resolve/faa77113e93335b63423e9e8d33395f86eaf8946/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + repeat_penalty: 1.05 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true - url: github:mudler/LocalAI/gallery/virtual.yaml@master name: adavar-3b-q4 description: | @@ -17836,6 +17916,8 @@ template: use_tokenizer_template: true variants: + - model: lfm2.5-24b-a2b-ud-q4 + - model: lfm2.5-24b-a2b-ud-q6 - model: lfm2.5-24b-a2b-q5 - model: lfm2.5-24b-a2b-q6 - model: lfm2.5-24b-a2b-q8 diff --git a/gallery/models/index.yaml b/gallery/models/index.yaml index da124ab3a..42c1000d9 100644 --- a/gallery/models/index.yaml +++ b/gallery/models/index.yaml @@ -4,6 +4,102 @@ # changes and never edit it by hand. CI fails when it is stale. A build from # master reads it as the default model source; a tagged build reads the gallery # until the release cutover. +- kind: model + id: lfm2.5-24b-a2b-ud-q4 + name: lfm2.5-24b-a2b-ud-q4 + version: 0.0.0-gallery.fbd02ecf6f42 + description: | + LiquidAI LFM2.5-24B-A2B is a text-only reasoning model with 24B total + parameters and 2.45B active parameters for multilingual chat and tool use. + This entry uses Unsloth's dynamic UD-Q4_K_XL GGUF quantization. + license: other + urls: + - https://huggingface.co/LiquidAI/LFM2.5-24B-A2B + - https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + sha256: a235d23638cc5549cb1ff61da7522c8a7e97f056f27093462a1e56f35aba91d8 + uri: https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF/resolve/faa77113e93335b63423e9e8d33395f86eaf8946/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q4_K_XL.gguf + repeat_penalty: 1.05 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true +- kind: model + id: lfm2.5-24b-a2b-ud-q6 + name: lfm2.5-24b-a2b-ud-q6 + version: 0.0.0-gallery.863d8f36da1e + description: | + LiquidAI LFM2.5-24B-A2B is a text-only reasoning model with 24B total + parameters and 2.45B active parameters for multilingual chat and tool use. + This entry uses Unsloth's dynamic UD-Q6_K_XL GGUF quantization. + license: other + urls: + - https://huggingface.co/LiquidAI/LFM2.5-24B-A2B + - https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF + tags: + - llm + - gguf + - reasoning + - cpu + - gpu + icon: https://cdn-uploads.huggingface.co/production/uploads/61b8e2ba285851687028d395/2b08LKpev0DNEk6DlnWkY.png + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + sha256: abcb7e21e006eb4c826a2b75fb91c0804db7b20f908a61beeebdbdd07c88f41d + uri: https://huggingface.co/unsloth/LFM2.5-24B-A2B-GGUF/resolve/faa77113e93335b63423e9e8d33395f86eaf8946/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + overrides: + backend: llama-cpp + context_size: 131072 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - completion + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/LFM2.5-24B-A2B-GGUF/LFM2.5-24B-A2B-UD-Q6_K_XL.gguf + repeat_penalty: 1.05 + temperature: 0.1 + top_k: 50 + template: + use_tokenizer_template: true - kind: model id: adavar-3b-q4 name: adavar-3b-q4 @@ -20813,6 +20909,8 @@ template: use_tokenizer_template: true variants: + - model: lfm2.5-24b-a2b-ud-q4 + - model: lfm2.5-24b-a2b-ud-q6 - model: lfm2.5-24b-a2b-q5 - model: lfm2.5-24b-a2b-q6 - model: lfm2.5-24b-a2b-q8