diff --git a/.docker/turboquant-build-target.sh b/.docker/turboquant-build-target.sh new file mode 100755 index 000000000..054c792d1 --- /dev/null +++ b/.docker/turboquant-build-target.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +arch=${1:?target architecture is required} +build_type=${2-} + +# GPU arm64 base images do not consistently provide the gcc-14 toolchain needed +# to compile ggml's armv9.2 CPU variants. Keep their portable fallback until the +# builder images can supply that compiler. +if [ "$arch" = "arm64" ] && [ -n "$build_type" ]; then + echo turboquant-fallback +else + echo turboquant-cpu-all +fi diff --git a/.docker/turboquant-compile.sh b/.docker/turboquant-compile.sh index ca6cf2690..b2ba5ffce 100755 --- a/.docker/turboquant-compile.sh +++ b/.docker/turboquant-compile.sh @@ -19,20 +19,18 @@ fi cd /LocalAI/backend/cpp/turboquant -if [ -z "${BUILD_TYPE:-}" ]; then - # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries. +BUILD_TARGET=$(/LocalAI/.docker/turboquant-build-target.sh "${TARGETARCH}" "${BUILD_TYPE:-}") +if [ "$BUILD_TARGET" = "turboquant-cpu-all" ]; then + # BUILD_TYPE remains in the environment, so GPU builds retain their accelerator while + # ggml selects the best CPU library when model work is offloaded to the host. # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme). if [ "${TARGETARCH}" = "arm64" ]; then + sh /LocalAI/.docker/apt-mirror.sh || true apt-get update -qq && apt-get install -y -qq gcc-14 g++-14 export CC=gcc-14 CXX=g++-14 fi - make turboquant-cpu-all -else - # GPU build (cublas/hipblas/sycl/vulkan/...): single fallback CPU build, the accelerator - # does the compute. Keeps the GPU compile from also building the CPU variant matrix and - # avoids the gcc-14 apt step on GPU base images such as nvidia l4t. - make turboquant-fallback fi +make "$BUILD_TARGET" make turboquant-grpc make turboquant-rpc-server diff --git a/backend/cpp/turboquant/run.sh b/backend/cpp/turboquant/run.sh index 869797faf..607a6f061 100755 --- a/backend/cpp/turboquant/run.sh +++ b/backend/cpp/turboquant/run.sh @@ -12,9 +12,11 @@ grep -e "flags" /proc/cpuinfo | head -1 BINARY=turboquant-fallback -# x86/arm64 ship a single turboquant-cpu-all built with ggml CPU_ALL_VARIANTS: ggml's +# CPU images and x86 GPU images ship a single turboquant-cpu-all built with ggml +# CPU_ALL_VARIANTS: ggml's # backend registry dlopens the best libggml-cpu-*.so for this host, so no shell-side -# probing. ROCm ships only turboquant-fallback, so fall back to it when cpu-all is absent. +# probing. GPU arm64 images still ship turboquant-fallback until their builder toolchains +# support ggml's complete arm variant matrix. if [ -e "$CURDIR"/turboquant-cpu-all ]; then BINARY=turboquant-cpu-all fi diff --git a/scripts/build/turboquant-build-target_test.sh b/scripts/build/turboquant-build-target_test.sh new file mode 100755 index 000000000..faebb1d70 --- /dev/null +++ b/scripts/build/turboquant-build-target_test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +CURDIR=$(dirname "$(realpath "$0")") +SELECTOR="$CURDIR/../../.docker/turboquant-build-target.sh" + +assert_target() { + local arch=$1 + local build_type=$2 + local expected=$3 + local actual + + actual=$("$SELECTOR" "$arch" "$build_type") + if [ "$actual" != "$expected" ]; then + echo "FAIL: $arch/$build_type selected $actual, expected $expected" + exit 1 + fi +} + +assert_target amd64 cublas turboquant-cpu-all +assert_target amd64 vulkan turboquant-cpu-all +assert_target amd64 "" turboquant-cpu-all +assert_target arm64 cublas turboquant-fallback +assert_target arm64 "" turboquant-cpu-all + +echo "PASS: turboquant build target preserves CPU variants where supported"