From cedcbf97a9df97a52aa0422902c1404b84fedb03 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot <bot-opensource@localaisrl.com>
Date: Sat, 1 Aug 2026 09:26:23 +0200
Subject: [PATCH] fix(llama-cpp): retain CPU variants in GPU builds (#11255)

Build the runtime CPU variant set alongside x86 GPU backends so partial offload uses the host's SIMD kernels instead of the scalar fallback. Keep arm64 GPU images on the portable binary until their builders consistently provide gcc-14.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
---
 .docker/llama-cpp-build-target.sh            | 14 +++++++++++
 .docker/llama-cpp-compile.sh                 | 18 ++++++--------
 backend/cpp/llama-cpp/run.sh                 |  8 +++---
 scripts/build/llama-cpp-build-target_test.sh | 26 ++++++++++++++++++++
 4 files changed, 51 insertions(+), 15 deletions(-)
 create mode 100755 .docker/llama-cpp-build-target.sh
 create mode 100755 scripts/build/llama-cpp-build-target_test.sh

diff --git a/.docker/llama-cpp-build-target.sh b/.docker/llama-cpp-build-target.sh
new file mode 100755
index 000000000..177fdc7ad
--- /dev/null
+++ b/.docker/llama-cpp-build-target.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+arch=${1:?target architecture is required}
+build_type=${2-}
+
+# GPU arm64 base images do not consistently provide the gcc-14 toolchain needed
+# to compile ggml's armv9.2 CPU variants. Keep their portable fallback until the
+# builder images can supply that compiler.
+if [ "$arch" = "arm64" ] && [ -n "$build_type" ]; then
+  echo llama-cpp-fallback
+else
+  echo llama-cpp-cpu-all
+fi
diff --git a/.docker/llama-cpp-compile.sh b/.docker/llama-cpp-compile.sh
index 112d43c16..32ff2a239 100755
--- a/.docker/llama-cpp-compile.sh
+++ b/.docker/llama-cpp-compile.sh
@@ -18,10 +18,12 @@ if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
 fi
 
 cd /LocalAI/backend/cpp/llama-cpp
-if [ -z "${BUILD_TYPE:-}" ]; then
-  # Pure CPU image (BUILD_TYPE empty): one build with ggml CPU_ALL_VARIANTS replaces the
-  # per-microarch binaries (x86: avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). ggml
-  # dlopens the best libggml-cpu-*.so at runtime by probing host CPU features.
+BUILD_TARGET=$(/LocalAI/.docker/llama-cpp-build-target.sh "${TARGETARCH}" "${BUILD_TYPE:-}")
+if [ "$BUILD_TARGET" = "llama-cpp-cpu-all" ]; then
+  # One build with ggml CPU_ALL_VARIANTS replaces the per-microarch binaries (x86:
+  # avx/avx2/avx512/fallback; arm64: armv8.x/armv9.x). BUILD_TYPE remains in the
+  # environment, so GPU builds retain their accelerator backend while ggml dlopens the
+  # best CPU library when work is offloaded to the host.
   #
   # arm64: the CPU_ALL_VARIANTS table includes armv9.2 SME variants whose -march=...+sme is
   # rejected by the Ubuntu 24.04 default gcc-13. gcc-14 accepts it, so build the arm64
@@ -35,14 +37,8 @@ if [ -z "${BUILD_TYPE:-}" ]; then
     apt-get update -qq && apt-get install -y -qq gcc-14 g++-14
     export CC=gcc-14 CXX=g++-14
   fi
-  make llama-cpp-cpu-all
-else
-  # GPU build (cublas/hipblas/sycl/vulkan/...): the accelerator does the compute, so a
-  # single fallback CPU build is enough - no per-microarch CPU variants needed. (This also
-  # keeps the heavy GPU backend compile from also building the whole CPU variant matrix,
-  # and avoids the gcc-14 apt step on GPU base images such as nvidia l4t.)
-  make llama-cpp-fallback
 fi
+make "$BUILD_TARGET"
 make llama-cpp-grpc
 make llama-cpp-rpc-server
 
diff --git a/backend/cpp/llama-cpp/run.sh b/backend/cpp/llama-cpp/run.sh
index 761098319..3182801f1 100755
--- a/backend/cpp/llama-cpp/run.sh
+++ b/backend/cpp/llama-cpp/run.sh
@@ -12,10 +12,10 @@ grep -e "flags" /proc/cpuinfo | head -1
 
 BINARY=llama-cpp-fallback
 
-# CPU images (x86, arm64, darwin) ship a single llama-cpp-cpu-all built with ggml
+# CPU images and x86 GPU images ship a single llama-cpp-cpu-all built with ggml
 # CPU_ALL_VARIANTS: ggml's backend registry dlopens the best libggml-cpu-*.so for this
-# host, so no shell-side AVX probing. GPU images (cublas/sycl/vulkan/hipblas) ship only
-# llama-cpp-fallback (the accelerator does the compute), so fall back to it when absent.
+# host, so no shell-side AVX probing. GPU arm64 images still ship llama-cpp-fallback
+# until their builder toolchains support ggml's complete arm variant matrix.
 if [ -e "$CURDIR"/llama-cpp-cpu-all ]; then
 	BINARY=llama-cpp-cpu-all
 fi
@@ -76,4 +76,4 @@ echo "Using binary: $BINARY"
 exec "$CURDIR"/$BINARY "$@"
 
 # We should never reach this point, however just in case we do, run fallback
-exec "$CURDIR"/llama-cpp-fallback "$@"
\ No newline at end of file
+exec "$CURDIR"/llama-cpp-fallback "$@"
diff --git a/scripts/build/llama-cpp-build-target_test.sh b/scripts/build/llama-cpp-build-target_test.sh
new file mode 100755
index 000000000..3d3970b55
--- /dev/null
+++ b/scripts/build/llama-cpp-build-target_test.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CURDIR=$(dirname "$(realpath "$0")")
+SELECTOR="$CURDIR/../../.docker/llama-cpp-build-target.sh"
+
+assert_target() {
+  local arch=$1
+  local build_type=$2
+  local expected=$3
+  local actual
+
+  actual=$("$SELECTOR" "$arch" "$build_type")
+  if [ "$actual" != "$expected" ]; then
+    echo "FAIL: $arch/$build_type selected $actual, expected $expected"
+    exit 1
+  fi
+}
+
+assert_target amd64 cublas llama-cpp-cpu-all
+assert_target amd64 vulkan llama-cpp-cpu-all
+assert_target amd64 "" llama-cpp-cpu-all
+assert_target arm64 cublas llama-cpp-fallback
+assert_target arm64 "" llama-cpp-cpu-all
+
+echo "PASS: llama.cpp build target preserves CPU variants where supported"