fix(cachyllama): link SSD cache implementation

CachyLLaMA splits persistent prompt-cache support into separate source files. Include those implementations in the monolithic LocalAI gRPC adapter when the fork provides them so the ARM64 fallback build resolves the page-manager symbols.

Assisted-by: Codex:gpt-5 [Codex]

.docker/cachyllama-compile.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Shared compile logic for backend/Dockerfile.cachyllama.
+# Sourced (via bind mount) from both builder-fromsource and builder-prebuilt stages.
+
+set -euxo pipefail
+
+export CCACHE_DIR=/root/.ccache
+ccache --max-size=5G || true
+ccache -z || true
+
+export CMAKE_ARGS="${CMAKE_ARGS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache"
+
+if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
+  CUDA_ARCH_ESC="${CUDA_DOCKER_ARCH//;/\\;}"
+  export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH_ESC}"
+  echo "CMAKE_ARGS(env) = ${CMAKE_ARGS}"
+  rm -rf /LocalAI/backend/cpp/cachyllama-*-build
+fi
+
+cd /LocalAI/backend/cpp/cachyllama
+
+if [ -z "${BUILD_TYPE:-}" ]; then
+  # Keep arm64 on the portable, fully linked build. CachyLLaMA's ARM
+  # CPU_ALL_VARIANTS build includes SME variants that do not build reliably
+  # across the Linux and Darwin toolchains used by backend CI.
+  if [ "${TARGETARCH}" = "arm64" ]; then
+    make cachyllama-fallback
+  else
+    # One ggml CPU_ALL_VARIANTS build replaces the per-microarch x86 binaries.
+    make cachyllama-cpu-all
+  fi
+else
+  # GPU build (cublas/hipblas/sycl/vulkan/...): single fallback CPU build, the accelerator
+  # does the compute. Keeps the GPU compile from also building the CPU variant matrix and
+  # avoids the gcc-14 apt step on GPU base images such as nvidia l4t.
+  make cachyllama-fallback
+fi
+make cachyllama-grpc
+make cachyllama-rpc-server
+
+ccache -s || true

.github/backend-matrix.yml

@@ -6176,6 +6176,68 @@ include:
     context: "./"
     ubuntu-version: '2404'
 
+  # CachyLLaMA targets the lower-spec/APU use case with CPU and Vulkan builds.
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+
 # Darwin matrix (consumed by backend-jobs-darwin).
 includeDarwin:
   - backend: "diffusers"
@@ -6202,6 +6264,9 @@ includeDarwin:
   - backend: "llama-cpp"
     tag-suffix: "-metal-darwin-arm64-llama-cpp"
     lang: "go"
+  - backend: "cachyllama"
+    tag-suffix: "-metal-darwin-arm64-cachyllama"
+    lang: "go"
   - backend: "stablediffusion-ggml"
     tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml"
     build-type: "metal"

.github/workflows/backend_build_darwin.yml

@@ -230,6 +230,12 @@ jobs:
           make protogen-go
           make backends/llama-cpp-darwin
 
+      - name: Build CachyLLaMA backend (Darwin Metal)
+        if: inputs.backend == 'cachyllama'
+        run: |
+          make protogen-go
+          make backends/cachyllama-darwin
+
       - name: Build ds4 backend (Darwin Metal)
         if: inputs.backend == 'ds4'
         run: |
@@ -256,7 +262,7 @@ jobs:
           make backends/audio-cpp-darwin
 
       - name: Build ${{ inputs.backend }}-darwin
-        if: inputs.backend != 'llama-cpp' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' && inputs.backend != 'audio-cpp'
+        if: inputs.backend != 'llama-cpp' && inputs.backend != 'cachyllama' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' && inputs.backend != 'audio-cpp'
         run: |
           make protogen-go
           BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend

.github/workflows/bump_deps.yaml

@@ -22,6 +22,10 @@ jobs:
             variable: "TURBOQUANT_VERSION"
             branch: "feature/turboquant-kv-cache"
             file: "backend/cpp/turboquant/Makefile"
+          - repository: "fewtarius/CachyLLama"
+            variable: "CACHYLLAMA_VERSION"
+            branch: "master"
+            file: "backend/cpp/cachyllama/Makefile"
           - repository: "PrismML-Eng/llama.cpp"
             variable: "BONSAI_VERSION"
             branch: "prism"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/cachyllama backends/cachyllama-darwin backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -1210,6 +1210,10 @@ backends/llama-cpp-darwin: build
 	bash ./scripts/build/llama-cpp-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)"
 
+backends/cachyllama-darwin: build
+	bash ./scripts/build/cachyllama-darwin.sh
+	./local-ai backends install "ocifile://$(abspath ./backend-images/cachyllama.tar)"
+
 backends/ds4-darwin: build
 	bash ./scripts/build/ds4-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/ds4.tar)"
@@ -1267,6 +1271,8 @@ BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 # turboquant is a llama.cpp fork with TurboQuant KV-cache quantization.
 # Reuses backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
 BACKEND_TURBOQUANT = turboquant|turboquant|.|false|false
+# CachyLLaMA is a llama.cpp fork with persistent SSD prompt caching and APU tuning.
+BACKEND_CACHYLLAMA = cachyllama|cachyllama|.|false|false
 # bonsai is a llama.cpp fork (PrismML) adding the Q1_0 (1-bit) and Q2_0 (ternary)
 # weight-quant kernels the Bonsai / Ternary-Bonsai models ship in. Reuses
 # backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
@@ -1384,6 +1390,7 @@ endef
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TURBOQUANT)))
+$(eval $(call generate-docker-build-target,$(BACKEND_CACHYLLAMA)))
 $(eval $(call generate-docker-build-target,$(BACKEND_BONSAI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DS4)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER)))
@@ -1456,7 +1463,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-cachyllama docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/Dockerfile.cachyllama

@@ -0,0 +1,160 @@
+ARG BASE_IMAGE=ubuntu:24.04
+# BUILDER_BASE_IMAGE defaults to BASE_IMAGE so the Dockerfile parses even
+# when no prebuilt base is supplied. The builder-prebuilt stage is only
+# entered when BUILDER_TARGET=builder-prebuilt, so a "wrong" fallback
+# content here is harmless — BuildKit prunes the unreferenced builder.
+ARG BUILDER_BASE_IMAGE=${BASE_IMAGE}
+# BUILDER_TARGET selects which builder stage the final scratch image copies
+# package output from. Declared at global scope (before any FROM) so it's
+# usable in `FROM ${BUILDER_TARGET}` below. Default keeps local
+# `make backends/cachyllama` on the from-source path.
+ARG BUILDER_TARGET=builder-fromsource
+ARG APT_MIRROR=""
+ARG APT_PORTS_MIRROR=""
+
+
+# ============================================================================
+# Stage: builder-fromsource — self-contained build path.
+# Runs .docker/install-base-deps.sh (apt deps + cmake + protoc + gRPC +
+# conditional CUDA/ROCm/Vulkan), copies /opt/grpc to /usr/local, then
+# compiles the variant. Used when BUILDER_TARGET=builder-fromsource (the
+# default; local `make backends/cachyllama`).
+#
+# The install script is the same one that backend/Dockerfile.base-grpc-builder
+# runs, so the result is bit-equivalent to the prebuilt-base path
+# (builder-prebuilt below).
+# ============================================================================
+FROM ${BASE_IMAGE} AS builder-fromsource
+ARG BUILD_TYPE
+ARG CUDA_MAJOR_VERSION
+ARG CUDA_MINOR_VERSION
+ARG CMAKE_FROM_SOURCE=false
+# CUDA Toolkit 13.x compatibility: CMake 3.31.9+ fixes toolchain detection/arch table issues
+ARG CMAKE_VERSION=3.31.10
+ARG GRPC_VERSION=v1.65.0
+ARG GRPC_MAKEFLAGS="-j4 -Otarget"
+ARG SKIP_DRIVERS=false
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG GO_VERSION=1.25.4
+ARG UBUNTU_VERSION=2404
+ARG APT_MIRROR
+ARG APT_PORTS_MIRROR
+ARG AMDGPU_TARGETS=""
+ARG BACKEND=rerankers
+# CUDA target archs, e.g. --build-arg CUDA_DOCKER_ARCH='75;86;89;120'
+ARG CUDA_DOCKER_ARCH
+ARG CMAKE_ARGS
+
+ENV BUILD_TYPE=${BUILD_TYPE} \
+    CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} \
+    CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION} \
+    CMAKE_FROM_SOURCE=${CMAKE_FROM_SOURCE} \
+    CMAKE_VERSION=${CMAKE_VERSION} \
+    GRPC_VERSION=${GRPC_VERSION} \
+    GRPC_MAKEFLAGS=${GRPC_MAKEFLAGS} \
+    SKIP_DRIVERS=${SKIP_DRIVERS} \
+    TARGETARCH=${TARGETARCH} \
+    UBUNTU_VERSION=${UBUNTU_VERSION} \
+    APT_MIRROR=${APT_MIRROR} \
+    APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \
+    AMDGPU_TARGETS=${AMDGPU_TARGETS} \
+    CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH} \
+    CMAKE_ARGS=${CMAKE_ARGS} \
+    DEBIAN_FRONTEND=noninteractive
+
+# CUDA on PATH (no-op when CUDA isn't installed)
+ENV PATH=/usr/local/cuda/bin:${PATH}
+# HipBLAS / ROCm on PATH (no-op when ROCm isn't installed)
+ENV PATH=/opt/rocm/bin:${PATH}
+
+WORKDIR /build
+
+# Install everything via the shared script — the same one that
+# backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and
+# this from-source path are bit-equivalent.
+RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \
+    --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+    bash /usr/local/sbin/install-base-deps
+
+# Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so
+# CMake's find_package finds it at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+# BuildKit cache mount for ccache. See Dockerfile.llama-cpp (commit 9228e5b4)
+# for rationale. cachyllama is a llama.cpp fork that reuses
+# backend/cpp/llama-cpp source via a thin wrapper Makefile, so MOST TUs
+# are content-identical to the upstream llama-cpp build. Sharing a cache
+# id with llama-cpp could give cross-fork hits — but for now keep them
+# separate so a regression in one doesn't poison the other. Revisit
+# sharing after measuring the actual hit rate.
+#
+# The compile body is shared with builder-prebuilt via .docker/cachyllama-compile.sh.
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+
+# Copy libraries using a script to handle architecture differences
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Stage: builder-prebuilt — uses the pre-built base from
+# quay.io/go-skynet/ci-cache:base-grpc-* (built by .github/workflows/base-images.yml).
+# That image already has gRPC at /opt/grpc + apt deps + CUDA/ROCm/Vulkan
+# pre-installed, so we just copy gRPC to /usr/local and compile. Used when
+# BUILDER_TARGET=builder-prebuilt (CI when the matrix entry sets
+# builder-base-image).
+# ============================================================================
+FROM ${BUILDER_BASE_IMAGE} AS builder-prebuilt
+
+ARG BUILD_TYPE
+ENV BUILD_TYPE=${BUILD_TYPE}
+ARG CUDA_DOCKER_ARCH
+ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
+ARG CMAKE_ARGS
+ENV CMAKE_ARGS=${CMAKE_ARGS}
+# AMDGPU_TARGETS must be forwarded into the env here too — backend/cpp/llama-cpp/Makefile
+# (which the cachyllama Makefile reuses via a sibling build dir) errors out when the var
+# is empty on a hipblas build, and the prebuilt path is what CI exercises most of the
+# time. The builder-fromsource stage above already does this; mirror it here.
+ARG AMDGPU_TARGETS
+ENV AMDGPU_TARGETS=${AMDGPU_TARGETS}
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# The base-grpc-* image installs gRPC to /opt/grpc but doesn't copy it to
+# /usr/local. Mirror what the from-source path does so the compile step
+# can find gRPC at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Final stage — copies package output from one of the two builders.
+# BUILDER_TARGET selects which one. BuildKit prunes the unreferenced builder.
+#
+# BuildKit doesn't support variable expansion in `COPY --from=` directly,
+# so we resolve the ARG by aliasing the chosen builder to a fixed stage
+# name via `FROM ${BUILDER_TARGET} AS builder` and then COPY --from=builder.
+# BUILDER_TARGET itself is declared as a global ARG at the top of this
+# file (required for use in FROM), so we just re-import it into this
+# stage's scope before the FROM directive.
+# ============================================================================
+FROM ${BUILDER_TARGET} AS builder
+
+FROM scratch
+
+
+# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
+COPY --from=builder /LocalAI/backend/cpp/cachyllama/package/. ./

backend/cpp/cachyllama/Makefile

@@ -0,0 +1,99 @@
+
+# Pinned to the HEAD of master on https://github.com/fewtarius/CachyLLama.
+# Auto-bumped nightly by .github/workflows/bump_deps.yaml.
+CACHYLLAMA_VERSION?=cea45f4222e1eea5c2c68388d7e75904a65b6778
+LLAMA_REPO?=https://github.com/fewtarius/CachyLLama
+
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
+TARGET?=--target grpc-server
+JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+ARCH?=$(shell uname -m)
+
+CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+LLAMA_CPP_DIR := $(CURRENT_MAKEFILE_DIR)/../llama-cpp
+
+GREEN := \033[0;32m
+RESET := \033[0m
+
+# cachyllama is a llama.cpp fork. Rather than duplicating grpc-server.cpp / CMakeLists.txt /
+# prepare.sh we reuse the ones in backend/cpp/llama-cpp, and only swap which repo+sha the
+# fetch step pulls. Each flavor target copies ../llama-cpp into a sibling ../cachyllama-<flavor>-build
+# directory, then invokes llama-cpp's own build-llama-cpp-grpc-server with LLAMA_REPO/LLAMA_VERSION
+# overridden to point at the fork.
+# Each flavor target:
+#   1. copies backend/cpp/llama-cpp/ (grpc-server.cpp + prepare.sh + CMakeLists.txt + Makefile)
+#      into a sibling cachyllama-<flavor>-build directory;
+#   2. clones the cachyllama fork into cachyllama-<flavor>-build/llama.cpp via the copy's
+#      own `llama.cpp` target, overriding LLAMA_REPO/LLAMA_VERSION;
+#   3. runs the copy's `grpc-server` target, which produces the binary we copy up as
+#      cachyllama-<flavor>.
+define cachyllama-build
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	# Drop patches vendored for upstream llama.cpp; CachyLLaMA tracks upstream
+	# closely but must not accidentally receive patches pinned to another SHA.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:$(1)$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build llama.cpp
+	CMAKE_ARGS="$(CMAKE_ARGS) $(2)" TARGET="$(3)" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server cachyllama-$(1)
+endef
+
+cachyllama-avx2:
+	$(call cachyllama-build,avx2,-DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx512:
+	$(call cachyllama-build,avx512,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx:
+	$(call cachyllama-build,avx,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+cachyllama-fallback:
+	$(call cachyllama-build,fallback,-DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+# Single-build CPU backend via ggml CPU_ALL_VARIANTS (mirrors llama-cpp-cpu-all).
+# cachyllama reuses backend/cpp/llama-cpp's CMakeLists.txt (hw_grpc_proto STATIC) and
+# Makefile (SHARED_LIBS make-var + EXTRA_CMAKE_ARGS), so this passes the same overrides
+# through to the copied build: SHARED_LIBS=ON, the DL flags, and --target ggml (which
+# pulls in the per-microarch libggml-cpu-*.so via ggml's add_dependencies). The .so set
+# is collected for package.sh to bundle into package/lib.
+cachyllama-cpu-all:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	# Drop patches vendored for upstream llama.cpp; they are SHA-specific.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:cpu-all-variants$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build llama.cpp
+	SHARED_LIBS=ON EXTRA_CMAKE_ARGS="-DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON" TARGET="--target grpc-server --target ggml" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server cachyllama-cpu-all
+	rm -rf ggml-shared-libs && mkdir -p ggml-shared-libs
+	find $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/llama.cpp/build \( -name '*.so*' -o -name '*.dylib' \) -exec cp -av {} ggml-shared-libs/ \;
+	@echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/
+
+cachyllama-grpc:
+	$(call cachyllama-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target ggml-rpc-server)
+
+cachyllama-rpc-server: cachyllama-grpc
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-grpc-build/llama.cpp/build/bin/ggml-rpc-server cachyllama-rpc-server
+
+package:
+	bash package.sh
+
+purge:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-*-build
+	rm -rf cachyllama-* package
+
+clean: purge

backend/cpp/cachyllama/package.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avrf $CURDIR/cachyllama-* $CURDIR/package/
+cp -rfv $CURDIR/run.sh $CURDIR/package/
+
+# Bundle the ggml shared backends from the CPU_ALL_VARIANTS build into package/lib. ggml
+# discovers the per-microarch libggml-cpu-*.so by scanning the executable directory, which
+# (via the bundled lib/ld.so that run.sh launches through) resolves to lib/. See the
+# matching comment in backend/cpp/llama-cpp/package.sh. No-op on the fallback/ROCm builds.
+if [ -d "$CURDIR/ggml-shared-libs" ]; then
+    echo "Bundling ggml shared backends (CPU_ALL_VARIANTS)..."
+    cp -avf $CURDIR/ggml-shared-libs/*.so* $CURDIR/package/lib/
+fi
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/

backend/cpp/cachyllama/run.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath "$0")")
+
+cd /
+
+echo "CPU info:"
+grep -e "model\sname" /proc/cpuinfo | head -1
+grep -e "flags" /proc/cpuinfo | head -1
+
+BINARY=cachyllama-fallback
+
+# x86/arm64 ship a single cachyllama-cpu-all built with ggml CPU_ALL_VARIANTS: ggml's
+# backend registry dlopens the best libggml-cpu-*.so for this host, so no shell-side
+# probing. ROCm ships only cachyllama-fallback, so fall back to it when cpu-all is absent.
+if [ -e "$CURDIR"/cachyllama-cpu-all ]; then
+	BINARY=cachyllama-cpu-all
+fi
+
+if [ -n "$LLAMACPP_GRPC_SERVERS" ]; then
+	if [ -e "$CURDIR"/cachyllama-grpc ]; then
+		BINARY=cachyllama-grpc
+	fi
+fi
+
+# Extend ld library path with the dir where this script is located/lib
+if [ "$(uname)" == "Darwin" ]; then
+	export DYLD_LIBRARY_PATH="$CURDIR"/lib:$DYLD_LIBRARY_PATH
+else
+	export LD_LIBRARY_PATH="$CURDIR"/lib:$LD_LIBRARY_PATH
+	# Tell rocBLAS where to find TensileLibrary data (GPU kernel tuning files)
+	if [ -d "$CURDIR/lib/rocblas/library" ]; then
+		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
+	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
+fi
+
+# If there is a lib/ld.so, use it
+if [ -f "$CURDIR"/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using binary: $BINARY"
+	exec "$CURDIR"/lib/ld.so "$CURDIR"/$BINARY "$@"
+fi
+
+echo "Using binary: $BINARY"
+exec "$CURDIR"/$BINARY "$@"
+
+# We should never reach this point, however just in case we do, run fallback
+exec "$CURDIR"/cachyllama-fallback "$@"

backend/cpp/llama-cpp/grpc-server.cpp

@@ -43,6 +43,15 @@
 #if __has_include("server-stream.cpp")
 #include "server-stream.cpp"
 #endif
+// CachyLLaMA splits its persistent prompt-cache implementation out of
+// server-context.cpp. Pull both implementation files into this monolithic
+// adapter TU when present; older llama.cpp pins do not ship them.
+#if __has_include("server-context-ssd-cache.cpp")
+#include "server-context-ssd-cache.cpp"
+#endif
+#if __has_include("server-context-page-manager.cpp")
+#include "server-context-page-manager.cpp"
+#endif
 #include "server-context.cpp"
 
 // LocalAI

backend/index.yaml

@@ -72,6 +72,27 @@
     nvidia-cuda-12: "cuda12-turboquant"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-turboquant"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-turboquant"
+- &cachyllama
+  name: "cachyllama"
+  alias: "cachyllama"
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: "cpu-cachyllama"
+    vulkan: "vulkan-cachyllama"
+    metal: "metal-cachyllama"
 - &bonsai
   name: "bonsai"
   alias: "bonsai"
@@ -2011,6 +2032,12 @@
     nvidia-cuda-12: "cuda12-turboquant-development"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-turboquant-development"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-turboquant-development"
+- !!merge <<: *cachyllama
+  name: "cachyllama-development"
+  capabilities:
+    default: "cpu-cachyllama-development"
+    vulkan: "vulkan-cachyllama-development"
+    metal: "metal-cachyllama-development"
 - !!merge <<: *bonsai
   name: "bonsai-development"
   capabilities:
@@ -2767,6 +2794,37 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant"
   mirrors:
     - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant
+## cachyllama
+- !!merge <<: *cachyllama
+  name: "cpu-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-cpu-cachyllama
+- !!merge <<: *cachyllama
+  name: "cpu-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-cpu-cachyllama
+- !!merge <<: *cachyllama
+  name: "vulkan-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-gpu-vulkan-cachyllama
+- !!merge <<: *cachyllama
+  name: "vulkan-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-gpu-vulkan-cachyllama
+- !!merge <<: *cachyllama
+  name: "metal-cachyllama"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-cachyllama"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-cachyllama
+- !!merge <<: *cachyllama
+  name: "metal-cachyllama-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-cachyllama"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-cachyllama
 ## bonsai
 - !!merge <<: *bonsai
   name: "cpu-bonsai"

core/gallery/importers/importers_test.go

@@ -302,7 +302,7 @@ var _ = Describe("DiscoverModelConfig", func() {
 				names = append(names, e.Name)
 				modalities = append(modalities, e.Modality)
 			}
-			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant"))
+			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant", "cachyllama"))
 			for _, m := range modalities {
 				Expect(m).To(Equal("text"))
 			}

core/gallery/importers/llama-cpp.go

@@ -37,6 +37,7 @@ func (i *LlamaCPPImporter) AdditionalBackends() []KnownBackendEntry {
 	return []KnownBackendEntry{
 		{Name: "ik-llama-cpp", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with ik-quants"},
 		{Name: "turboquant", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with TurboQuant optimizations"},
+		{Name: "cachyllama", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with persistent SSD prompt caching and APU tuning"},
 		{Name: "vllm-cpp", Modality: "text", Description: "vLLM-style continuous-batching engine (vllm.cpp) consuming GGUF, by the LocalAI team"},
 	}
 }
@@ -136,7 +137,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	backend := "llama-cpp"
 	if b, ok := preferencesMap["backend"].(string); ok {
 		switch b {
-		case "ik-llama-cpp", "turboquant", "vllm-cpp":
+		case "ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp":
 			backend = b
 		}
 	}

core/gallery/importers/llama-cpp_test.go

@@ -181,6 +181,23 @@ var _ = Describe("LlamaCPPImporter", func() {
 			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
 		})
 
+		It("swaps the emitted backend to cachyllama when preferred", func() {
+			preferences := json.RawMessage(`{"backend": "cachyllama"}`)
+			details := Details{
+				URI:         "https://example.com/my-model.gguf",
+				Preferences: preferences,
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: cachyllama"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("backend: llama-cpp\n"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("model: my-model.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(len(modelConfig.Files)).To(Equal(1))
+			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
+		})
+
 		It("swaps the emitted backend to vllm-cpp when preferred, keeping engine-side templating", func() {
 			preferences := json.RawMessage(`{"backend": "vllm-cpp"}`)
 			details := Details{
@@ -551,7 +568,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 	})
 
 	Context("AdditionalBackends", func() {
-		It("advertises ik-llama-cpp, turboquant and vllm-cpp as drop-in replacements", func() {
+		It("advertises the curated GGUF drop-in replacements", func() {
 			entries := importer.AdditionalBackends()
 
 			names := make([]string, 0, len(entries))
@@ -560,7 +577,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 				names = append(names, e.Name)
 				byName[e.Name] = e
 			}
-			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "vllm-cpp"))
+			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp"))
 
 			for _, name := range names {
 				e := byName[name]

core/http/react-ui/e2e/import-form-ux-batch-d.spec.js

@@ -14,6 +14,7 @@ const MOCK_BACKENDS = [
   { name: 'llama-cpp', modality: 'text', auto_detect: true, installed: true },
   { name: 'ik-llama-cpp', modality: 'text', auto_detect: true, installed: true },
   { name: 'turboquant', modality: 'text', auto_detect: true, installed: true },
+  { name: 'cachyllama', modality: 'text', auto_detect: true, installed: true },
   { name: 'stablediffusion-ggml', modality: 'image', auto_detect: true, installed: true },
   { name: 'transformers', modality: 'text', auto_detect: true, installed: true },
   { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true },
@@ -92,6 +93,14 @@ test.describe('Import form UX — Batch D (progressive disclosure)', () => {
     await expect(modelTypeInput(page)).toHaveCount(0)
   })
 
+  test('D1 — selecting cachyllama shows GGUF Quantizations + MMProj', async ({ page }) => {
+    await enterPowerPreferences(page)
+    await selectBackend(page, 'cachyllama')
+    await expect(quantizationsInput(page)).toBeVisible()
+    await expect(mmprojInput(page)).toBeVisible()
+    await expect(modelTypeInput(page)).toHaveCount(0)
+  })
+
   test('D1 — selecting transformers hides Quantizations + MMProj, shows Model Type', async ({ page }) => {
     await enterPowerPreferences(page)
     await selectBackend(page, 'transformers')

core/http/react-ui/src/pages/ImportModel.jsx

@@ -334,11 +334,11 @@ export default function ImportModel() {
   // flipping backends back and forth doesn't lose input.
   const showQuantizations = useMemo(() => {
     if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'stablediffusion-ggml'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama', 'stablediffusion-ggml'].includes(prefs.backend)
   }, [prefs.backend])
   const showMmprojQuantizations = useMemo(() => {
     if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama'].includes(prefs.backend)
   }, [prefs.backend])
   const showModelType = useMemo(() => {
     if (!prefs.backend) return true

docs/content/advanced/model-configuration.md

@@ -629,7 +629,7 @@ These options apply when using the `vllm` backend:
 | `disable_log_stats` | bool | Disable logging statistics |
 | `dtype` | string | Data type (e.g., `float16`, `bfloat16`) |
 | `flash_attention` | string | Flash attention configuration |
-| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
+| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`, `cachyllama`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
 | `cache_type_v` | string | Value cache quantization type. Maps to llama.cpp's `-ctv`. Same accepted values as `cache_type_k`. Note: any quantized V cache requires flash_attention to be enabled. |
 | `limit_mm_per_prompt` | object | Limit multimodal content per prompt: `{image: int, video: int, audio: int}` |
 

docs/content/features/text-generation.md

@@ -684,6 +684,53 @@ The `cache_type_k` / `cache_type_v` fields map to llama.cpp's `-ctk` / `-ctv` fl
 - [llama-cpp-turboquant](https://github.com/TheTom/llama-cpp-turboquant)
 - [Tracked branch: `feature/turboquant-kv-cache`](https://github.com/TheTom/llama-cpp-turboquant/tree/feature/turboquant-kv-cache)
 
+### CachyLLaMA (persistent prompt cache for lower-spec hardware)
+
+[CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) is a llama.cpp fork aimed at
+APUs, integrated GPUs, handhelds, and other shared-memory or lower-throughput
+systems. Its main addition is an SSD-backed KV cache that restores stable prompt
+prefixes across requests and process restarts.
+
+Install the alias and select it in model YAML like any other GGUF backend:
+
+```bash
+local-ai backends install cachyllama
+```
+
+```yaml
+name: cached-agent-model
+backend: cachyllama
+parameters:
+  model: model.gguf
+options:
+  - --cache-ssd:/models/.cachyllama-cache
+  - --cache-ssd-checkpoints:64
+  - --cache-ssd-hot-window:16384
+  - --cache-ssd-warm-window:32768
+  - --cache-ssd-page-size:1024
+  - --cache-ssd-system-prompts:8
+  - --cache-ssd-system-max-days:30
+```
+
+The cache directory must be on persistent storage and writable by LocalAI.
+Inside a container, put it below a mounted models or data volume.
+
+| Option | Meaning |
+|--------|---------|
+| `--cache-ssd` | Directory for SSD-backed KV checkpoints; enables the feature. |
+| `--cache-ssd-checkpoints` | Maximum checkpoints kept per slot. |
+| `--cache-ssd-hot-window` | Token window always retained in the hot tier. |
+| `--cache-ssd-warm-window` | Token window retained in RAM when possible. |
+| `--cache-ssd-max-cold` | Maximum cold checkpoints; `0` means unlimited. |
+| `--cache-ssd-page-size` | Tokens per cache page: `512`, `1024`, or `2048`. |
+| `--cache-ssd-max-conversations` | Maximum conversation directories; `0` means unlimited. |
+| `--cache-ssd-system-prompts` | Number of cross-conversation system prompts to cache. |
+| `--cache-ssd-system-max-days` | Expire unused system-prompt entries after this many days. |
+
+LocalAI publishes CPU and Vulkan images for Linux on amd64 and arm64, plus a
+Metal image for Apple silicon. The Vulkan build is intended for AMD APUs and
+other supported integrated GPUs.
+
 
 ### vLLM
 

docs/content/reference/compatibility-table.md

@@ -23,6 +23,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({
 | [llama.cpp](https://github.com/ggerganov/llama.cpp) | LLM inference in C/C++. Supports LLaMA, Mamba, RWKV, Falcon, Starcoder, GPT-2, [and many others](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#description) | GPT, Functions | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
 | [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) | Hard fork of llama.cpp optimized for CPU/hybrid CPU+GPU with IQK quants, custom quant mixes, and MLA for DeepSeek | GPT | yes | yes | CPU (AVX2+) |
 | [turboquant](https://github.com/TheTom/llama-cpp-turboquant) | llama.cpp fork adding the TurboQuant KV-cache quantization scheme | GPT | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Jetson L4T |
+| [CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) | llama.cpp fork with persistent SSD-backed prompt caching and APU tuning | GPT | yes | yes | CPU, Vulkan, Metal |
 | [ds4](https://github.com/antirez/ds4) | DeepSeek V4 Flash single-model inference engine, optimized for Metal and CUDA | GPT | no | yes | CPU, CUDA 12/13, Metal, Jetson L4T |
 | [vllm.cpp](https://github.com/mudler/vllm.cpp) | From-scratch C++20 port of vLLM by the LocalAI team: paged KV cache, continuous batching, prefix caching, safetensors + GGUF, engine-enforced structured output, no Python at inference | GPT, Functions | no | yes | CPU, CUDA 12/13 (Blackwell-family), Vulkan, Metal, Jetson L4T (GB10) |
 | [vLLM](https://github.com/vllm-project/vllm) | Fast LLM serving with PagedAttention; GPTQ/AWQ/FP8 quantization | GPT, Functions, Multimodal | no | yes | CUDA 12/13, ROCm, Intel SYCL, Jetson L4T |

scripts/build/cachyllama-darwin.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="${IMAGE_NAME:-localai/cachyllama-darwin}"
+
+pushd backend/cpp/cachyllama
+
+# CachyLLaMA's ARM CPU_ALL_VARIANTS build includes SME variants that do not build
+# reliably across the Darwin toolchains used by backend CI. The fallback build
+# remains Metal-enabled on Darwin and is fully linked.
+make cachyllama-fallback && \
+make cachyllama-grpc && \
+make cachyllama-rpc-server
+
+popd
+
+mkdir -p build/darwin
+mkdir -p backend-images
+mkdir -p build/darwin/lib
+
+cp -rf backend/cpp/cachyllama/cachyllama-fallback build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-grpc build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-rpc-server build/darwin/
+
+# Set default additional libs only for Darwin on M chips (arm64)
+if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-$(ls /opt/homebrew/Cellar/protobuf/**/lib/libutf8_validity*.dylib 2>/dev/null)}
+else
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-""}
+fi
+
+for file in $ADDITIONAL_LIBS; do
+  cp -rfv $file build/darwin/lib
+done
+
+for file in build/darwin/*; do
+  LIBS="$(otool -L $file | awk 'NR > 1 { system("echo " $1) } ' | xargs echo)"
+  for lib in $LIBS; do
+    # only libraries ending in dylib
+    if [[ "$lib" == *.dylib ]]; then
+        if [ -e "$lib" ]; then
+            cp -rvf "$lib" build/darwin/lib
+        fi
+    fi
+  done
+done
+
+echo "--------------------------------"
+echo "ADDITIONAL_LIBS: $ADDITIONAL_LIBS"
+echo "--------------------------------"
+
+echo "Bundled libraries:"
+ls -la build/darwin/lib
+
+
+cp -rf backend/cpp/cachyllama/run.sh build/darwin/
+
+PLATFORMARCH="${PLATFORMARCH:-darwin/arm64}"
+
+./local-ai util create-oci-image \
+        build/darwin/. \
+        --output ./backend-images/cachyllama.tar \
+        --image-name $IMAGE_NAME \
+        --platform $PLATFORMARCH
+
+rm -rf build/darwin

scripts/lib/backend-filter.mjs

@@ -70,6 +70,10 @@ export function inferBackendPath(item) {
     // via a thin wrapper Makefile. Changes to either dir should retrigger it.
     return `backend/cpp/turboquant/`;
   }
+  if (item.dockerfile.endsWith("cachyllama")) {
+    // CachyLLaMA is a llama.cpp fork that reuses the LocalAI gRPC sources.
+    return `backend/cpp/cachyllama/`;
+  }
   if (item.dockerfile.endsWith("bonsai")) {
     // bonsai is a llama.cpp fork that reuses backend/cpp/llama-cpp sources
     // via a thin wrapper Makefile. Changes to either dir should retrigger it.
@@ -97,6 +101,9 @@ export function inferBackendPathDarwin(item) {
   if (item.backend === "llama-cpp") {
     return `backend/cpp/llama-cpp/`;
   }
+  if (item.backend === "cachyllama") {
+    return `backend/cpp/cachyllama/`;
+  }
   // ds4 is C++ too (built via `make backends/ds4-darwin`); the matrix entry
   // carries lang=go for runner/toolchain selection, but the source is C++.
   if (item.backend === "ds4") {
@@ -144,7 +151,7 @@ export function backendChanged(backend, pathPrefix, changedFiles) {
 
   // Fork backends reuse backend/cpp/llama-cpp sources via thin wrappers;
   // changes to either directory must retrigger their pipelines.
-  return (backend === "turboquant" || backend === "bonsai") &&
+  return (backend === "turboquant" || backend === "cachyllama" || backend === "bonsai") &&
     changedFiles.some(file => file.startsWith("backend/cpp/llama-cpp/"));
 }
 
@@ -152,12 +159,13 @@ export function backendChanged(backend, pathPrefix, changedFiles) {
 // without it is a Python backend (see .github/backend-matrix.yml).
 const isDarwinPython = item => !item.lang;
 
-// backend_build_darwin.yml routes llama-cpp, ds4, privacy-filter and audio-cpp
+// backend_build_darwin.yml routes llama-cpp, cachyllama, ds4, privacy-filter and audio-cpp
 // to their own bespoke make targets; every other lang=go entry goes through
 // `make build-darwin-go-backend` -> scripts/build/golang-darwin.sh. Keep this
 // set in sync with the `if:` conditions in that workflow.
 const DARWIN_BESPOKE_BUILDERS = new Set([
   "llama-cpp",
+  "cachyllama",
   "ds4",
   "privacy-filter",
   "audio-cpp",
@@ -410,6 +418,16 @@ export const SHARED_BUILD_INPUTS = [
     linux: never,
     darwin: item => item.backend === "llama-cpp",
   },
+  {
+    matches: file => file === "scripts/build/cachyllama-darwin.sh",
+    linux: never,
+    darwin: item => item.backend === "cachyllama",
+  },
+  {
+    matches: file => file === ".docker/cachyllama-compile.sh",
+    linux: item => item.backend === "cachyllama",
+    darwin: never,
+  },
   {
     matches: file => file === "scripts/build/ds4-darwin.sh",
     linux: never,

scripts/lib/backend-filter_test.mjs

@@ -71,6 +71,12 @@ const includes = [
     "tag-suffix": "-cpu-audio-cpp",
     "base-image": "ubuntu:24.04",
   },
+  {
+    backend: "cachyllama",
+    dockerfile: "./backend/Dockerfile.cachyllama",
+    "tag-suffix": "-cachyllama",
+    "base-image": "ubuntu:24.04",
+  },
 ];
 
 const includesDarwin = [
@@ -78,6 +84,7 @@ const includesDarwin = [
   { backend: "mlx", "tag-suffix": "-metal-darwin-arm64-mlx", "build-type": "mps" },
   { backend: "whisper", lang: "go", "tag-suffix": "-metal-darwin-arm64-whisper", "build-type": "metal" },
   { backend: "llama-cpp", lang: "go", "tag-suffix": "-metal-darwin-arm64-llama-cpp", "build-type": "metal" },
+  { backend: "cachyllama", lang: "go", "tag-suffix": "-metal-darwin-arm64-cachyllama", "build-type": "metal" },
   { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" },
 ];
 
@@ -190,6 +197,19 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => {
   assert.deepEqual(names(filteredDarwin), ["ds4"]);
 });
 
+test("the CachyLLaMA Darwin build script rebuilds only CachyLLaMA", () => {
+  const { filteredDarwin } = run(["scripts/build/cachyllama-darwin.sh"]);
+
+  assert.deepEqual(names(filteredDarwin), ["cachyllama"]);
+});
+
+test("the CachyLLaMA compile helper rebuilds only Linux CachyLLaMA", () => {
+  const { filtered, filteredDarwin } = run([".docker/cachyllama-compile.sh"]);
+
+  assert.deepEqual(names(filtered), ["cachyllama"]);
+  assert.deepEqual(filteredDarwin, []);
+});
+
 test("an unclassified scripts/build/ file conservatively rebuilds everything", () => {
   const { filtered, filteredDarwin } = run([
     "scripts/build/package-something-new.sh",
@@ -211,7 +231,17 @@ test("tests for the packaging scripts do not rebuild anything", () => {
 test("turboquant still retriggers on llama-cpp source changes", () => {
   const { filtered } = run(["backend/cpp/llama-cpp/grpc-server.cpp"]);
 
-  assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]);
+  assert.deepEqual(names(filtered), ["cachyllama", "llama-cpp", "turboquant"]);
+});
+
+test("cachyllama maps to its wrapper source directory", () => {
+  assert.equal(
+    inferBackendPath({
+      backend: "cachyllama",
+      dockerfile: "./backend/Dockerfile.cachyllama",
+    }),
+    "backend/cpp/cachyllama/"
+  );
 });
 
 // ---------------------------------------------------------------------------