diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 3bdfc6d41..e45267f41 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -153,6 +153,28 @@ MTP speculative decoding is not enabled by these entries. The weights use [PolyForm Small Business 1.0.0 with a personal-use grant](https://huggingface.co/bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF/blob/main/LICENSE). Review that license for permitted use. +## AstaBrief-8B + +[AstaBrief-8B](https://huggingface.co/allenai/AstaBrief_8B) is Ai2's Qwen3-8B +fine-tune for writing cited reports from scientific literature excerpts. +Install the llama.cpp GGUF entry with: + +```bash +local-ai models install astabrief-8b-q4 +``` + +The entry offers Q4_K_M and Q8_0 builds. To select a specific build, use +`--variant astabrief-8b-q4` or `--variant astabrief-8b-q8` during installation. +Both use the embedded chat template, a 32,768-token context, temperature `0.7`, +and top-p `0.95`. + +Use the upstream [research prompt](https://huggingface.co/datasets/allenai/AstaBrief_prompts/blob/main/sft_prompt.txt). +Replace `[QUERY]` with your question and `[SECTION_REFERENCES]` with the cited +excerpts, then send the completed prompt as a user message to +`/v1/chat/completions`. The model writes from the material you supply; this +gallery entry does not search for or retrieve papers. The upstream example +uses `max_tokens: 4096` for the report. + ## Hemmingway-1 Install `hemmingway-1` for English text generation with llama.cpp. The gallery groups its Q4_K_M and Q8_0 builds as variants. diff --git a/gallery/index.yaml b/gallery/index.yaml index 9e52e70eb..28dded8d7 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -27226,6 +27226,92 @@ - filename: Zero-Qwen3-8B-OpenZero-Q5_K_M.gguf sha256: 390464f750b5cb53da298848adc05839c1fd40404a74cd5f800cad9612d17d59 uri: huggingface://shafire/Zero-Qwen3-8B-OpenZero-GGUF/Zero-Qwen3-8B-OpenZero-Q5_K_M.gguf +- name: astabrief-8b-q4 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/allenai/AstaBrief_8B + - https://huggingface.co/mradermacher/AstaBrief_8B-GGUF + description: | + AstaBrief-8B is Ai2's Qwen3-8B fine-tune for writing cited research reports + from a question and supplied scientific literature excerpts. This Q4_K_M + GGUF build uses llama.cpp and the embedded chat template. Provide the + upstream research prompt and references in the user message; the model + does not retrieve papers itself. + license: apache-2.0 + tags: + - llm + - gguf + - cpu + - gpu + - qwen3 + - research + - en + variants: + - model: astabrief-8b-q8 + overrides: + backend: llama-cpp + known_usecases: + - chat + context_size: 32768 + mmap: true + f16: true + options: + - use_jinja:true + template: + use_tokenizer_template: true + parameters: + model: llama-cpp/models/astabrief-8b/AstaBrief_8B.Q4_K_M.gguf + temperature: 0.7 + top_p: 0.95 + top_k: 0 + stopwords: + - <|im_end|> + files: + - filename: llama-cpp/models/astabrief-8b/AstaBrief_8B.Q4_K_M.gguf + uri: https://huggingface.co/mradermacher/AstaBrief_8B-GGUF/resolve/a40b9eb85c6e349898737726b4b9ee69688b63a3/AstaBrief_8B.Q4_K_M.gguf + sha256: 1df1265b9715fc5118134010abcd3d297fd4be08a9b2294c7a467c79f161ccce +- name: astabrief-8b-q8 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/allenai/AstaBrief_8B + - https://huggingface.co/mradermacher/AstaBrief_8B-GGUF + description: | + AstaBrief-8B is Ai2's Qwen3-8B fine-tune for writing cited research reports + from a question and supplied scientific literature excerpts. This Q8_0 + GGUF build uses llama.cpp and the embedded chat template. Provide the + upstream research prompt and references in the user message; the model + does not retrieve papers itself. + license: apache-2.0 + tags: + - llm + - gguf + - cpu + - gpu + - qwen3 + - research + - en + overrides: + backend: llama-cpp + known_usecases: + - chat + context_size: 32768 + mmap: true + f16: true + options: + - use_jinja:true + template: + use_tokenizer_template: true + parameters: + model: llama-cpp/models/astabrief-8b/AstaBrief_8B.Q8_0.gguf + temperature: 0.7 + top_p: 0.95 + top_k: 0 + stopwords: + - <|im_end|> + files: + - filename: llama-cpp/models/astabrief-8b/AstaBrief_8B.Q8_0.gguf + uri: https://huggingface.co/mradermacher/AstaBrief_8B-GGUF/resolve/a40b9eb85c6e349898737726b4b9ee69688b63a3/AstaBrief_8B.Q8_0.gguf + sha256: c199cbac2cccdbe16e6ca3aed60c9868216ed1ed90a157f2451de37bba074d2a - name: qwen3-8b url: github:mudler/LocalAI/gallery/qwen3.yaml@master urls: