feat(backends): add CachyLLaMA support

Add CachyLLaMA as a GGUF-compatible llama.cpp fork backend with CPU and Vulkan builds on Linux and Metal on Apple silicon. Expose it through model import, document persistent SSD cache options, and wire CI path filtering and dependency updates. Assisted-by: Codex:gpt-5
2026-07-30 01:48:06 -04:00 · 2026-07-29 18:15:33 +00:00
parent 9bfd71387b
commit c4b5ed846f
21 changed files with 777 additions and 13 deletions
--- a/.docker/cachyllama-compile.sh
+++ b/.docker/cachyllama-compile.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Shared compile logic for backend/Dockerfile.cachyllama.
+# Sourced (via bind mount) from both builder-fromsource and builder-prebuilt stages.
+
+set -euxo pipefail
+
+export CCACHE_DIR=/root/.ccache
+ccache --max-size=5G || true
+ccache -z || true
+
+export CMAKE_ARGS="${CMAKE_ARGS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache"
+
+if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
+  CUDA_ARCH_ESC="${CUDA_DOCKER_ARCH//;/\\;}"
+  export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH_ESC}"
+  echo "CMAKE_ARGS(env) = ${CMAKE_ARGS}"
+  rm -rf /LocalAI/backend/cpp/cachyllama-*-build
+fi
+
+cd /LocalAI/backend/cpp/cachyllama
+
+if [ -z "${BUILD_TYPE:-}" ]; then
+  # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries.
+  # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme).
+  if [ "${TARGETARCH}" = "arm64" ]; then
+    apt-get update -qq && apt-get install -y -qq gcc-14 g++-14
+    export CC=gcc-14 CXX=g++-14
+  fi
+  make cachyllama-cpu-all
+else
+  # GPU build (cublas/hipblas/sycl/vulkan/...): single fallback CPU build, the accelerator
+  # does the compute. Keeps the GPU compile from also building the CPU variant matrix and
+  # avoids the gcc-14 apt step on GPU base images such as nvidia l4t.
+  make cachyllama-fallback
+fi
+make cachyllama-grpc
+make cachyllama-rpc-server
+
+ccache -s || true
--- a/.github/backend-matrix.yml
+++ b/.github/backend-matrix.yml
@@ -6085,6 +6085,68 @@ include:
    context: "./"
    ubuntu-version: '2404'

+  # CachyLLaMA targets the lower-spec/APU use case with CPU and Vulkan builds.
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+
 # Darwin matrix (consumed by backend-jobs-darwin).
 includeDarwin:
  - backend: "diffusers"
@@ -6111,6 +6173,9 @@ includeDarwin:
  - backend: "llama-cpp"
    tag-suffix: "-metal-darwin-arm64-llama-cpp"
    lang: "go"
+  - backend: "cachyllama"
+    tag-suffix: "-metal-darwin-arm64-cachyllama"
+    lang: "go"
  - backend: "stablediffusion-ggml"
    tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml"
    build-type: "metal"
--- a/.github/workflows/backend_build_darwin.yml
+++ b/.github/workflows/backend_build_darwin.yml
@@ -230,6 +230,12 @@ jobs:
          make protogen-go
          make backends/llama-cpp-darwin

+      - name: Build CachyLLaMA backend (Darwin Metal)
+        if: inputs.backend == 'cachyllama'
+        run: |
+          make protogen-go
+          make backends/cachyllama-darwin
+
      - name: Build ds4 backend (Darwin Metal)
        if: inputs.backend == 'ds4'
        run: |
@@ -245,7 +251,7 @@ jobs:
          make backends/privacy-filter-darwin

      - name: Build ${{ inputs.backend }}-darwin
-        if: inputs.backend != 'llama-cpp' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter'
+        if: inputs.backend != 'llama-cpp' && inputs.backend != 'cachyllama' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter'
        run: |
          make protogen-go
          BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend
--- a/.github/workflows/bump_deps.yaml
+++ b/.github/workflows/bump_deps.yaml
@@ -22,6 +22,10 @@ jobs:
            variable: "TURBOQUANT_VERSION"
            branch: "feature/turboquant-kv-cache"
            file: "backend/cpp/turboquant/Makefile"
+          - repository: "fewtarius/CachyLLama"
+            variable: "CACHYLLAMA_VERSION"
+            branch: "master"
+            file: "backend/cpp/cachyllama/Makefile"
          - repository: "PrismML-Eng/llama.cpp"
            variable: "BONSAI_VERSION"
            branch: "prism"
--- a/11
+++ b/11
@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/cachyllama backends/cachyllama-darwin backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin

 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -1193,6 +1193,10 @@ backends/llama-cpp-darwin: build
 	bash ./scripts/build/llama-cpp-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)"

+backends/cachyllama-darwin: build
+	bash ./scripts/build/cachyllama-darwin.sh
+	./local-ai backends install "ocifile://$(abspath ./backend-images/cachyllama.tar)"
+
 backends/ds4-darwin: build
 	bash ./scripts/build/ds4-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/ds4.tar)"
@@ -1246,6 +1250,8 @@ BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 # turboquant is a llama.cpp fork with TurboQuant KV-cache quantization.
 # Reuses backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
 BACKEND_TURBOQUANT = turboquant|turboquant|.|false|false
+# CachyLLaMA is a llama.cpp fork with persistent SSD prompt caching and APU tuning.
+BACKEND_CACHYLLAMA = cachyllama|cachyllama|.|false|false
 # bonsai is a llama.cpp fork (PrismML) adding the Q1_0 (1-bit) and Q2_0 (ternary)
 # weight-quant kernels the Bonsai / Ternary-Bonsai models ship in. Reuses
 # backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
@@ -1358,6 +1364,7 @@ endef
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TURBOQUANT)))
+$(eval $(call generate-docker-build-target,$(BACKEND_CACHYLLAMA)))
 $(eval $(call generate-docker-build-target,$(BACKEND_BONSAI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DS4)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER)))
@@ -1429,7 +1436,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar

-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-cachyllama docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store

 ########################################################
 ### Mock Backend for E2E Tests
--- a/backend/Dockerfile.cachyllama
+++ b/backend/Dockerfile.cachyllama
@@ -0,0 +1,160 @@
+ARG BASE_IMAGE=ubuntu:24.04
+# BUILDER_BASE_IMAGE defaults to BASE_IMAGE so the Dockerfile parses even
+# when no prebuilt base is supplied. The builder-prebuilt stage is only
+# entered when BUILDER_TARGET=builder-prebuilt, so a "wrong" fallback
+# content here is harmless — BuildKit prunes the unreferenced builder.
+ARG BUILDER_BASE_IMAGE=${BASE_IMAGE}
+# BUILDER_TARGET selects which builder stage the final scratch image copies
+# package output from. Declared at global scope (before any FROM) so it's
+# usable in `FROM ${BUILDER_TARGET}` below. Default keeps local
+# `make backends/cachyllama` on the from-source path.
+ARG BUILDER_TARGET=builder-fromsource
+ARG APT_MIRROR=""
+ARG APT_PORTS_MIRROR=""
+
+
+# ============================================================================
+# Stage: builder-fromsource — self-contained build path.
+# Runs .docker/install-base-deps.sh (apt deps + cmake + protoc + gRPC +
+# conditional CUDA/ROCm/Vulkan), copies /opt/grpc to /usr/local, then
+# compiles the variant. Used when BUILDER_TARGET=builder-fromsource (the
+# default; local `make backends/cachyllama`).
+#
+# The install script is the same one that backend/Dockerfile.base-grpc-builder
+# runs, so the result is bit-equivalent to the prebuilt-base path
+# (builder-prebuilt below).
+# ============================================================================
+FROM ${BASE_IMAGE} AS builder-fromsource
+ARG BUILD_TYPE
+ARG CUDA_MAJOR_VERSION
+ARG CUDA_MINOR_VERSION
+ARG CMAKE_FROM_SOURCE=false
+# CUDA Toolkit 13.x compatibility: CMake 3.31.9+ fixes toolchain detection/arch table issues
+ARG CMAKE_VERSION=3.31.10
+ARG GRPC_VERSION=v1.65.0
+ARG GRPC_MAKEFLAGS="-j4 -Otarget"
+ARG SKIP_DRIVERS=false
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG GO_VERSION=1.25.4
+ARG UBUNTU_VERSION=2404
+ARG APT_MIRROR
+ARG APT_PORTS_MIRROR
+ARG AMDGPU_TARGETS=""
+ARG BACKEND=rerankers
+# CUDA target archs, e.g. --build-arg CUDA_DOCKER_ARCH='75;86;89;120'
+ARG CUDA_DOCKER_ARCH
+ARG CMAKE_ARGS
+
+ENV BUILD_TYPE=${BUILD_TYPE} \
+    CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} \
+    CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION} \
+    CMAKE_FROM_SOURCE=${CMAKE_FROM_SOURCE} \
+    CMAKE_VERSION=${CMAKE_VERSION} \
+    GRPC_VERSION=${GRPC_VERSION} \
+    GRPC_MAKEFLAGS=${GRPC_MAKEFLAGS} \
+    SKIP_DRIVERS=${SKIP_DRIVERS} \
+    TARGETARCH=${TARGETARCH} \
+    UBUNTU_VERSION=${UBUNTU_VERSION} \
+    APT_MIRROR=${APT_MIRROR} \
+    APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \
+    AMDGPU_TARGETS=${AMDGPU_TARGETS} \
+    CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH} \
+    CMAKE_ARGS=${CMAKE_ARGS} \
+    DEBIAN_FRONTEND=noninteractive
+
+# CUDA on PATH (no-op when CUDA isn't installed)
+ENV PATH=/usr/local/cuda/bin:${PATH}
+# HipBLAS / ROCm on PATH (no-op when ROCm isn't installed)
+ENV PATH=/opt/rocm/bin:${PATH}
+
+WORKDIR /build
+
+# Install everything via the shared script — the same one that
+# backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and
+# this from-source path are bit-equivalent.
+RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \
+    --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+    bash /usr/local/sbin/install-base-deps
+
+# Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so
+# CMake's find_package finds it at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+# BuildKit cache mount for ccache. See Dockerfile.llama-cpp (commit 9228e5b4)
+# for rationale. cachyllama is a llama.cpp fork that reuses
+# backend/cpp/llama-cpp source via a thin wrapper Makefile, so MOST TUs
+# are content-identical to the upstream llama-cpp build. Sharing a cache
+# id with llama-cpp could give cross-fork hits — but for now keep them
+# separate so a regression in one doesn't poison the other. Revisit
+# sharing after measuring the actual hit rate.
+#
+# The compile body is shared with builder-prebuilt via .docker/cachyllama-compile.sh.
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+
+# Copy libraries using a script to handle architecture differences
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Stage: builder-prebuilt — uses the pre-built base from
+# quay.io/go-skynet/ci-cache:base-grpc-* (built by .github/workflows/base-images.yml).
+# That image already has gRPC at /opt/grpc + apt deps + CUDA/ROCm/Vulkan
+# pre-installed, so we just copy gRPC to /usr/local and compile. Used when
+# BUILDER_TARGET=builder-prebuilt (CI when the matrix entry sets
+# builder-base-image).
+# ============================================================================
+FROM ${BUILDER_BASE_IMAGE} AS builder-prebuilt
+
+ARG BUILD_TYPE
+ENV BUILD_TYPE=${BUILD_TYPE}
+ARG CUDA_DOCKER_ARCH
+ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
+ARG CMAKE_ARGS
+ENV CMAKE_ARGS=${CMAKE_ARGS}
+# AMDGPU_TARGETS must be forwarded into the env here too — backend/cpp/llama-cpp/Makefile
+# (which the cachyllama Makefile reuses via a sibling build dir) errors out when the var
+# is empty on a hipblas build, and the prebuilt path is what CI exercises most of the
+# time. The builder-fromsource stage above already does this; mirror it here.
+ARG AMDGPU_TARGETS
+ENV AMDGPU_TARGETS=${AMDGPU_TARGETS}
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# The base-grpc-* image installs gRPC to /opt/grpc but doesn't copy it to
+# /usr/local. Mirror what the from-source path does so the compile step
+# can find gRPC at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Final stage — copies package output from one of the two builders.
+# BUILDER_TARGET selects which one. BuildKit prunes the unreferenced builder.
+#
+# BuildKit doesn't support variable expansion in `COPY --from=` directly,
+# so we resolve the ARG by aliasing the chosen builder to a fixed stage
+# name via `FROM ${BUILDER_TARGET} AS builder` and then COPY --from=builder.
+# BUILDER_TARGET itself is declared as a global ARG at the top of this
+# file (required for use in FROM), so we just re-import it into this
+# stage's scope before the FROM directive.
+# ============================================================================
+FROM ${BUILDER_TARGET} AS builder
+
+FROM scratch
+
+
+# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
+COPY --from=builder /LocalAI/backend/cpp/cachyllama/package/. ./
--- a/backend/cpp/cachyllama/Makefile
+++ b/backend/cpp/cachyllama/Makefile
@@ -0,0 +1,99 @@
+
+# Pinned to the HEAD of master on https://github.com/fewtarius/CachyLLama.
+# Auto-bumped nightly by .github/workflows/bump_deps.yaml.
+CACHYLLAMA_VERSION?=cea45f4222e1eea5c2c68388d7e75904a65b6778
+LLAMA_REPO?=https://github.com/fewtarius/CachyLLama
+
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
+TARGET?=--target grpc-server
+JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+ARCH?=$(shell uname -m)
+
+CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+LLAMA_CPP_DIR := $(CURRENT_MAKEFILE_DIR)/../llama-cpp
+
+GREEN := \033[0;32m
+RESET := \033[0m
+
+# cachyllama is a llama.cpp fork. Rather than duplicating grpc-server.cpp / CMakeLists.txt /
+# prepare.sh we reuse the ones in backend/cpp/llama-cpp, and only swap which repo+sha the
+# fetch step pulls. Each flavor target copies ../llama-cpp into a sibling ../cachyllama-<flavor>-build
+# directory, then invokes llama-cpp's own build-llama-cpp-grpc-server with LLAMA_REPO/LLAMA_VERSION
+# overridden to point at the fork.
+# Each flavor target:
+#   1. copies backend/cpp/llama-cpp/ (grpc-server.cpp + prepare.sh + CMakeLists.txt + Makefile)
+#      into a sibling cachyllama-<flavor>-build directory;
+#   2. clones the cachyllama fork into cachyllama-<flavor>-build/llama.cpp via the copy's
+#      own `llama.cpp` target, overriding LLAMA_REPO/LLAMA_VERSION;
+#   3. runs the copy's `grpc-server` target, which produces the binary we copy up as
+#      cachyllama-<flavor>.
+define cachyllama-build
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	# Drop patches vendored for upstream llama.cpp; CachyLLaMA tracks upstream
+	# closely but must not accidentally receive patches pinned to another SHA.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:$(1)$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build llama.cpp
+	CMAKE_ARGS="$(CMAKE_ARGS) $(2)" TARGET="$(3)" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server cachyllama-$(1)
+endef
+
+cachyllama-avx2:
+	$(call cachyllama-build,avx2,-DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx512:
+	$(call cachyllama-build,avx512,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx:
+	$(call cachyllama-build,avx,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+cachyllama-fallback:
+	$(call cachyllama-build,fallback,-DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+# Single-build CPU backend via ggml CPU_ALL_VARIANTS (mirrors llama-cpp-cpu-all).
+# cachyllama reuses backend/cpp/llama-cpp's CMakeLists.txt (hw_grpc_proto STATIC) and
+# Makefile (SHARED_LIBS make-var + EXTRA_CMAKE_ARGS), so this passes the same overrides
+# through to the copied build: SHARED_LIBS=ON, the DL flags, and --target ggml (which
+# pulls in the per-microarch libggml-cpu-*.so via ggml's add_dependencies). The .so set
+# is collected for package.sh to bundle into package/lib.
+cachyllama-cpu-all:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	# Drop patches vendored for upstream llama.cpp; they are SHA-specific.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:cpu-all-variants$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build llama.cpp
+	SHARED_LIBS=ON EXTRA_CMAKE_ARGS="-DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON" TARGET="--target grpc-server --target ggml" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server cachyllama-cpu-all
+	rm -rf ggml-shared-libs && mkdir -p ggml-shared-libs
+	find $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/llama.cpp/build \( -name '*.so*' -o -name '*.dylib' \) -exec cp -av {} ggml-shared-libs/ \;
+	@echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/
+
+cachyllama-grpc:
+	$(call cachyllama-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target ggml-rpc-server)
+
+cachyllama-rpc-server: cachyllama-grpc
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-grpc-build/llama.cpp/build/bin/ggml-rpc-server cachyllama-rpc-server
+
+package:
+	bash package.sh
+
+purge:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-*-build
+	rm -rf cachyllama-* package
+
+clean: purge
--- a/backend/cpp/cachyllama/package.sh
+++ b/backend/cpp/cachyllama/package.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avrf $CURDIR/cachyllama-* $CURDIR/package/
+cp -rfv $CURDIR/run.sh $CURDIR/package/
+
+# Bundle the ggml shared backends from the CPU_ALL_VARIANTS build into package/lib. ggml
+# discovers the per-microarch libggml-cpu-*.so by scanning the executable directory, which
+# (via the bundled lib/ld.so that run.sh launches through) resolves to lib/. See the
+# matching comment in backend/cpp/llama-cpp/package.sh. No-op on the fallback/ROCm builds.
+if [ -d "$CURDIR/ggml-shared-libs" ]; then
+    echo "Bundling ggml shared backends (CPU_ALL_VARIANTS)..."
+    cp -avf $CURDIR/ggml-shared-libs/*.so* $CURDIR/package/lib/
+fi
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/
--- a/backend/cpp/cachyllama/run.sh
+++ b/backend/cpp/cachyllama/run.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath "$0")")
+
+cd /
+
+echo "CPU info:"
+grep -e "model\sname" /proc/cpuinfo | head -1
+grep -e "flags" /proc/cpuinfo | head -1
+
+BINARY=cachyllama-fallback
+
+# x86/arm64 ship a single cachyllama-cpu-all built with ggml CPU_ALL_VARIANTS: ggml's
+# backend registry dlopens the best libggml-cpu-*.so for this host, so no shell-side
+# probing. ROCm ships only cachyllama-fallback, so fall back to it when cpu-all is absent.
+if [ -e "$CURDIR"/cachyllama-cpu-all ]; then
+	BINARY=cachyllama-cpu-all
+fi
+
+if [ -n "$LLAMACPP_GRPC_SERVERS" ]; then
+	if [ -e "$CURDIR"/cachyllama-grpc ]; then
+		BINARY=cachyllama-grpc
+	fi
+fi
+
+# Extend ld library path with the dir where this script is located/lib
+if [ "$(uname)" == "Darwin" ]; then
+	export DYLD_LIBRARY_PATH="$CURDIR"/lib:$DYLD_LIBRARY_PATH
+else
+	export LD_LIBRARY_PATH="$CURDIR"/lib:$LD_LIBRARY_PATH
+	# Tell rocBLAS where to find TensileLibrary data (GPU kernel tuning files)
+	if [ -d "$CURDIR/lib/rocblas/library" ]; then
+		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
+	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
+fi
+
+# If there is a lib/ld.so, use it
+if [ -f "$CURDIR"/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using binary: $BINARY"
+	exec "$CURDIR"/lib/ld.so "$CURDIR"/$BINARY "$@"
+fi
+
+echo "Using binary: $BINARY"
+exec "$CURDIR"/$BINARY "$@"
+
+# We should never reach this point, however just in case we do, run fallback
+exec "$CURDIR"/cachyllama-fallback "$@"
--- a/backend/index.yaml
+++ b/backend/index.yaml
@@ -72,6 +72,27 @@
    nvidia-cuda-12: "cuda12-turboquant"
    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-turboquant"
    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-turboquant"
+- &cachyllama
+  name: "cachyllama"
+  alias: "cachyllama"
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: "cpu-cachyllama"
+    vulkan: "vulkan-cachyllama"
+    metal: "metal-cachyllama"
 - &bonsai
  name: "bonsai"
  alias: "bonsai"
@@ -1979,6 +2000,12 @@
    nvidia-cuda-12: "cuda12-turboquant-development"
    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-turboquant-development"
    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-turboquant-development"
+- !!merge <<: *cachyllama
+  name: "cachyllama-development"
+  capabilities:
+    default: "cpu-cachyllama-development"
+    vulkan: "vulkan-cachyllama-development"
+    metal: "metal-cachyllama-development"
 - !!merge <<: *bonsai
  name: "bonsai-development"
  capabilities:
@@ -2726,6 +2753,37 @@
  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant"
  mirrors:
    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant
+## cachyllama
+- !!merge <<: *cachyllama
+  name: "cpu-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-cpu-cachyllama
+- !!merge <<: *cachyllama
+  name: "cpu-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-cpu-cachyllama
+- !!merge <<: *cachyllama
+  name: "vulkan-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-gpu-vulkan-cachyllama
+- !!merge <<: *cachyllama
+  name: "vulkan-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-gpu-vulkan-cachyllama
+- !!merge <<: *cachyllama
+  name: "metal-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-cachyllama
+- !!merge <<: *cachyllama
+  name: "metal-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-cachyllama
 ## bonsai
 - !!merge <<: *bonsai
  name: "cpu-bonsai"
--- a/core/gallery/importers/importers_test.go
+++ b/core/gallery/importers/importers_test.go
@@ -302,7 +302,7 @@ var _ = Describe("DiscoverModelConfig", func() {
 				names = append(names, e.Name)
 				modalities = append(modalities, e.Modality)
 			}
-			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant"))
+			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant", "cachyllama"))
 			for _, m := range modalities {
 				Expect(m).To(Equal("text"))
 			}
--- a/core/gallery/importers/llama-cpp.go
+++ b/core/gallery/importers/llama-cpp.go
@@ -37,6 +37,7 @@ func (i *LlamaCPPImporter) AdditionalBackends() []KnownBackendEntry {
 	return []KnownBackendEntry{
 		{Name: "ik-llama-cpp", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with ik-quants"},
 		{Name: "turboquant", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with TurboQuant optimizations"},
+		{Name: "cachyllama", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with persistent SSD prompt caching and APU tuning"},
 		{Name: "vllm-cpp", Modality: "text", Description: "vLLM-style continuous-batching engine (vllm.cpp) consuming GGUF, by the LocalAI team"},
 	}
 }
@@ -136,7 +137,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	backend := "llama-cpp"
 	if b, ok := preferencesMap["backend"].(string); ok {
 		switch b {
-		case "ik-llama-cpp", "turboquant", "vllm-cpp":
+		case "ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp":
 			backend = b
 		}
 	}
--- a/core/gallery/importers/llama-cpp_test.go
+++ b/core/gallery/importers/llama-cpp_test.go
@@ -181,6 +181,23 @@ var _ = Describe("LlamaCPPImporter", func() {
 			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
 		})

+		It("swaps the emitted backend to cachyllama when preferred", func() {
+			preferences := json.RawMessage(`{"backend": "cachyllama"}`)
+			details := Details{
+				URI:         "https://example.com/my-model.gguf",
+				Preferences: preferences,
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: cachyllama"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("backend: llama-cpp\n"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("model: my-model.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(len(modelConfig.Files)).To(Equal(1))
+			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
+		})
+
 		It("swaps the emitted backend to vllm-cpp when preferred, keeping engine-side templating", func() {
 			preferences := json.RawMessage(`{"backend": "vllm-cpp"}`)
 			details := Details{
@@ -551,7 +568,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 	})

 	Context("AdditionalBackends", func() {
-		It("advertises ik-llama-cpp, turboquant and vllm-cpp as drop-in replacements", func() {
+		It("advertises the curated GGUF drop-in replacements", func() {
 			entries := importer.AdditionalBackends()

 			names := make([]string, 0, len(entries))
@@ -560,7 +577,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 				names = append(names, e.Name)
 				byName[e.Name] = e
 			}
-			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "vllm-cpp"))
+			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp"))

 			for _, name := range names {
 				e := byName[name]
--- a/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
+++ b/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
@@ -14,6 +14,7 @@ const MOCK_BACKENDS = [
  { name: 'llama-cpp', modality: 'text', auto_detect: true, installed: true },
  { name: 'ik-llama-cpp', modality: 'text', auto_detect: true, installed: true },
  { name: 'turboquant', modality: 'text', auto_detect: true, installed: true },
+  { name: 'cachyllama', modality: 'text', auto_detect: true, installed: true },
  { name: 'stablediffusion-ggml', modality: 'image', auto_detect: true, installed: true },
  { name: 'transformers', modality: 'text', auto_detect: true, installed: true },
  { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true },
@@ -92,6 +93,14 @@ test.describe('Import form UX — Batch D (progressive disclosure)', () => {
    await expect(modelTypeInput(page)).toHaveCount(0)
  })

+  test('D1 — selecting cachyllama shows GGUF Quantizations + MMProj', async ({ page }) => {
+    await enterPowerPreferences(page)
+    await selectBackend(page, 'cachyllama')
+    await expect(quantizationsInput(page)).toBeVisible()
+    await expect(mmprojInput(page)).toBeVisible()
+    await expect(modelTypeInput(page)).toHaveCount(0)
+  })
+
  test('D1 — selecting transformers hides Quantizations + MMProj, shows Model Type', async ({ page }) => {
    await enterPowerPreferences(page)
    await selectBackend(page, 'transformers')
--- a/core/http/react-ui/src/pages/ImportModel.jsx
+++ b/core/http/react-ui/src/pages/ImportModel.jsx
@@ -336,11 +336,11 @@ export default function ImportModel() {
  // flipping backends back and forth doesn't lose input.
  const showQuantizations = useMemo(() => {
    if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'stablediffusion-ggml'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama', 'stablediffusion-ggml'].includes(prefs.backend)
  }, [prefs.backend])
  const showMmprojQuantizations = useMemo(() => {
    if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama'].includes(prefs.backend)
  }, [prefs.backend])
  const showModelType = useMemo(() => {
    if (!prefs.backend) return true
--- a/docs/content/advanced/model-configuration.md
+++ b/docs/content/advanced/model-configuration.md
@@ -629,7 +629,7 @@ These options apply when using the `vllm` backend:
 | `disable_log_stats` | bool | Disable logging statistics |
 | `dtype` | string | Data type (e.g., `float16`, `bfloat16`) |
 | `flash_attention` | string | Flash attention configuration |
-| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
+| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`, `cachyllama`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
 | `cache_type_v` | string | Value cache quantization type. Maps to llama.cpp's `-ctv`. Same accepted values as `cache_type_k`. Note: any quantized V cache requires flash_attention to be enabled. |
 | `limit_mm_per_prompt` | object | Limit multimodal content per prompt: `{image: int, video: int, audio: int}` |

--- a/docs/content/features/text-generation.md
+++ b/docs/content/features/text-generation.md
@@ -684,6 +684,53 @@ The `cache_type_k` / `cache_type_v` fields map to llama.cpp's `-ctk` / `-ctv` fl
 - [llama-cpp-turboquant](https://github.com/TheTom/llama-cpp-turboquant)
 - [Tracked branch: `feature/turboquant-kv-cache`](https://github.com/TheTom/llama-cpp-turboquant/tree/feature/turboquant-kv-cache)

+### CachyLLaMA (persistent prompt cache for lower-spec hardware)
+
+[CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) is a llama.cpp fork aimed at
+APUs, integrated GPUs, handhelds, and other shared-memory or lower-throughput
+systems. Its main addition is an SSD-backed KV cache that restores stable prompt
+prefixes across requests and process restarts.
+
+Install the alias and select it in model YAML like any other GGUF backend:
+
+```bash
+local-ai backends install cachyllama
+```
+
+```yaml
+name: cached-agent-model
+backend: cachyllama
+parameters:
+  model: model.gguf
+options:
+  - --cache-ssd:/models/.cachyllama-cache
+  - --cache-ssd-checkpoints:64
+  - --cache-ssd-hot-window:16384
+  - --cache-ssd-warm-window:32768
+  - --cache-ssd-page-size:1024
+  - --cache-ssd-system-prompts:8
+  - --cache-ssd-system-max-days:30
+```
+
+The cache directory must be on persistent storage and writable by LocalAI.
+Inside a container, put it below a mounted models or data volume.
+
+| Option | Meaning |
+|--------|---------|
+| `--cache-ssd` | Directory for SSD-backed KV checkpoints; enables the feature. |
+| `--cache-ssd-checkpoints` | Maximum checkpoints kept per slot. |
+| `--cache-ssd-hot-window` | Token window always retained in the hot tier. |
+| `--cache-ssd-warm-window` | Token window retained in RAM when possible. |
+| `--cache-ssd-max-cold` | Maximum cold checkpoints; `0` means unlimited. |
+| `--cache-ssd-page-size` | Tokens per cache page: `512`, `1024`, or `2048`. |
+| `--cache-ssd-max-conversations` | Maximum conversation directories; `0` means unlimited. |
+| `--cache-ssd-system-prompts` | Number of cross-conversation system prompts to cache. |
+| `--cache-ssd-system-max-days` | Expire unused system-prompt entries after this many days. |
+
+LocalAI publishes CPU and Vulkan images for Linux on amd64 and arm64, plus a
+Metal image for Apple silicon. The Vulkan build is intended for AMD APUs and
+other supported integrated GPUs.
+

 ### vLLM

--- a/docs/content/reference/compatibility-table.md
+++ b/docs/content/reference/compatibility-table.md
@@ -23,6 +23,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({
 | [llama.cpp](https://github.com/ggerganov/llama.cpp) | LLM inference in C/C++. Supports LLaMA, Mamba, RWKV, Falcon, Starcoder, GPT-2, [and many others](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#description) | GPT, Functions | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
 | [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) | Hard fork of llama.cpp optimized for CPU/hybrid CPU+GPU with IQK quants, custom quant mixes, and MLA for DeepSeek | GPT | yes | yes | CPU (AVX2+) |
 | [turboquant](https://github.com/TheTom/llama-cpp-turboquant) | llama.cpp fork adding the TurboQuant KV-cache quantization scheme | GPT | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Jetson L4T |
+| [CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) | llama.cpp fork with persistent SSD-backed prompt caching and APU tuning | GPT | yes | yes | CPU, Vulkan, Metal |
 | [ds4](https://github.com/antirez/ds4) | DeepSeek V4 Flash single-model inference engine, optimized for Metal and CUDA | GPT | no | yes | CPU, CUDA 12/13, Metal, Jetson L4T |
 | [vllm.cpp](https://github.com/mudler/vllm.cpp) | From-scratch C++20 port of vLLM by the LocalAI team: paged KV cache, continuous batching, prefix caching, safetensors + GGUF, engine-enforced structured output, no Python at inference | GPT, Functions | no | yes | CPU, CUDA 12/13 (Blackwell-family), Vulkan, Metal, Jetson L4T (GB10) |
 | [vLLM](https://github.com/vllm-project/vllm) | Fast LLM serving with PagedAttention; GPTQ/AWQ/FP8 quantization | GPT, Functions, Multimodal | no | yes | CUDA 12/13, ROCm, Intel SYCL, Jetson L4T |
--- a/scripts/build/cachyllama-darwin.sh
+++ b/scripts/build/cachyllama-darwin.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="${IMAGE_NAME:-localai/cachyllama-darwin}"
+
+pushd backend/cpp/cachyllama
+
+# Single build via ggml CPU_ALL_VARIANTS: one binary plus the per-microarch Apple/arm
+# dylibs (apple_m1/m2_m3/m4, armv8.x) that ggml selects at runtime. GGML_METAL stays ON
+# and --target ggml also builds ggml-metal (via add_dependencies), so the Metal GPU
+# backend is still produced as a loadable libggml-metal.dylib.
+make cachyllama-cpu-all && \
+make cachyllama-grpc && \
+make cachyllama-rpc-server
+
+popd
+
+mkdir -p build/darwin
+mkdir -p backend-images
+mkdir -p build/darwin/lib
+
+cp -rf backend/cpp/cachyllama/cachyllama-cpu-all build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-grpc build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-rpc-server build/darwin/
+
+# Distribute the shared ggml/llama libraries from the CPU_ALL_VARIANTS build. Unlike the
+# old fully-static fallback build, these have @rpath install names, so the otool loop below
+# (which only copies deps that exist on disk) will not pick them up. The split is by suffix:
+#  - ggml emits its loadable backends (per-microarch CPU variants, metal, blas) with a .so
+#    suffix EVEN ON DARWIN. These go in the package ROOT next to the binary, because darwin
+#    run.sh execs the binary directly (no bundled ld.so) so ggml's executable-directory
+#    scan looks there.
+#  - the core libraries (libggml-base/libggml/libllama/libllama-common/libmtmd) use the
+#    platform .dylib suffix and are NEEDED deps; they go in lib/, resolved at load time via
+#    the DYLD_LIBRARY_PATH=lib that run.sh exports. -a preserves the version symlinks.
+SHLIBS=backend/cpp/cachyllama/ggml-shared-libs
+cp -a $SHLIBS/*.so build/darwin/
+cp -a $SHLIBS/*.dylib build/darwin/lib/
+
+# Set default additional libs only for Darwin on M chips (arm64)
+if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-$(ls /opt/homebrew/Cellar/protobuf/**/lib/libutf8_validity*.dylib 2>/dev/null)}
+else
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-""}
+fi
+
+for file in $ADDITIONAL_LIBS; do
+  cp -rfv $file build/darwin/lib
+done
+
+for file in build/darwin/*; do
+  LIBS="$(otool -L $file | awk 'NR > 1 { system("echo " $1) } ' | xargs echo)"
+  for lib in $LIBS; do
+    # only libraries ending in dylib
+    if [[ "$lib" == *.dylib ]]; then
+        if [ -e "$lib" ]; then
+            cp -rvf "$lib" build/darwin/lib
+        fi
+    fi
+  done
+done
+
+echo "--------------------------------"
+echo "ADDITIONAL_LIBS: $ADDITIONAL_LIBS"
+echo "--------------------------------"
+
+echo "Bundled libraries:"
+ls -la build/darwin/lib
+
+
+cp -rf backend/cpp/cachyllama/run.sh build/darwin/
+
+PLATFORMARCH="${PLATFORMARCH:-darwin/arm64}"
+
+./local-ai util create-oci-image \
+        build/darwin/. \
+        --output ./backend-images/cachyllama.tar \
+        --image-name $IMAGE_NAME \
+        --platform $PLATFORMARCH
+
+rm -rf build/darwin
--- a/scripts/lib/backend-filter.mjs
+++ b/scripts/lib/backend-filter.mjs
@@ -70,6 +70,10 @@ export function inferBackendPath(item) {
    // via a thin wrapper Makefile. Changes to either dir should retrigger it.
    return `backend/cpp/turboquant/`;
  }
+  if (item.dockerfile.endsWith("cachyllama")) {
+    // CachyLLaMA is a llama.cpp fork that reuses the LocalAI gRPC sources.
+    return `backend/cpp/cachyllama/`;
+  }
  if (item.dockerfile.endsWith("bonsai")) {
    // bonsai is a llama.cpp fork that reuses backend/cpp/llama-cpp sources
    // via a thin wrapper Makefile. Changes to either dir should retrigger it.
@@ -94,6 +98,9 @@ export function inferBackendPathDarwin(item) {
  if (item.backend === "llama-cpp") {
    return `backend/cpp/llama-cpp/`;
  }
+  if (item.backend === "cachyllama") {
+    return `backend/cpp/cachyllama/`;
+  }
  // ds4 is C++ too (built via `make backends/ds4-darwin`); the matrix entry
  // carries lang=go for runner/toolchain selection, but the source is C++.
  if (item.backend === "ds4") {
@@ -134,7 +141,7 @@ export function backendChanged(backend, pathPrefix, changedFiles) {

  // Fork backends reuse backend/cpp/llama-cpp sources via thin wrappers;
  // changes to either directory must retrigger their pipelines.
-  return (backend === "turboquant" || backend === "bonsai") &&
+  return (backend === "turboquant" || backend === "cachyllama" || backend === "bonsai") &&
    changedFiles.some(file => file.startsWith("backend/cpp/llama-cpp/"));
 }

@@ -145,7 +152,7 @@ const isDarwinPython = item => !item.lang;
 // backend_build_darwin.yml routes llama-cpp, ds4 and privacy-filter to their
 // own bespoke make targets; every other lang=go entry goes through
 // `make build-darwin-go-backend` -> scripts/build/golang-darwin.sh.
-const DARWIN_BESPOKE_BUILDERS = new Set(["llama-cpp", "ds4", "privacy-filter"]);
+const DARWIN_BESPOKE_BUILDERS = new Set(["llama-cpp", "cachyllama", "ds4", "privacy-filter"]);
 const isDarwinGenericGo = item =>
  !!item.lang && !DARWIN_BESPOKE_BUILDERS.has(item.backend);

@@ -394,6 +401,16 @@ export const SHARED_BUILD_INPUTS = [
    linux: never,
    darwin: item => item.backend === "llama-cpp",
  },
+  {
+    matches: file => file === "scripts/build/cachyllama-darwin.sh",
+    linux: never,
+    darwin: item => item.backend === "cachyllama",
+  },
+  {
+    matches: file => file === ".docker/cachyllama-compile.sh",
+    linux: item => item.backend === "cachyllama",
+    darwin: never,
+  },
  {
    matches: file => file === "scripts/build/ds4-darwin.sh",
    linux: never,
--- a/scripts/lib/backend-filter_test.mjs
+++ b/scripts/lib/backend-filter_test.mjs
@@ -61,6 +61,12 @@ const includes = [
    "tag-suffix": "-turboquant",
    "base-image": "ubuntu:24.04",
  },
+  {
+    backend: "cachyllama",
+    dockerfile: "./backend/Dockerfile.cachyllama",
+    "tag-suffix": "-cachyllama",
+    "base-image": "ubuntu:24.04",
+  },
 ];

 const includesDarwin = [
@@ -68,6 +74,7 @@ const includesDarwin = [
  { backend: "mlx", "tag-suffix": "-metal-darwin-arm64-mlx", "build-type": "mps" },
  { backend: "whisper", lang: "go", "tag-suffix": "-metal-darwin-arm64-whisper", "build-type": "metal" },
  { backend: "llama-cpp", lang: "go", "tag-suffix": "-metal-darwin-arm64-llama-cpp", "build-type": "metal" },
+  { backend: "cachyllama", lang: "go", "tag-suffix": "-metal-darwin-arm64-cachyllama", "build-type": "metal" },
  { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" },
 ];

@@ -180,6 +187,19 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => {
  assert.deepEqual(names(filteredDarwin), ["ds4"]);
 });

+test("the CachyLLaMA Darwin build script rebuilds only CachyLLaMA", () => {
+  const { filteredDarwin } = run(["scripts/build/cachyllama-darwin.sh"]);
+
+  assert.deepEqual(names(filteredDarwin), ["cachyllama"]);
+});
+
+test("the CachyLLaMA compile helper rebuilds only Linux CachyLLaMA", () => {
+  const { filtered, filteredDarwin } = run([".docker/cachyllama-compile.sh"]);
+
+  assert.deepEqual(names(filtered), ["cachyllama"]);
+  assert.deepEqual(filteredDarwin, []);
+});
+
 test("an unclassified scripts/build/ file conservatively rebuilds everything", () => {
  const { filtered, filteredDarwin } = run([
    "scripts/build/package-something-new.sh",
@@ -201,7 +221,17 @@ test("tests for the packaging scripts do not rebuild anything", () => {
 test("turboquant still retriggers on llama-cpp source changes", () => {
  const { filtered } = run(["backend/cpp/llama-cpp/grpc-server.cpp"]);

-  assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]);
+  assert.deepEqual(names(filtered), ["cachyllama", "llama-cpp", "turboquant"]);
+});
+
+test("cachyllama maps to its wrapper source directory", () => {
+  assert.equal(
+    inferBackendPath({
+      backend: "cachyllama",
+      dockerfile: "./backend/Dockerfile.cachyllama",
+    }),
+    "backend/cpp/cachyllama/"
+  );
 });

 // ---------------------------------------------------------------------------