mirror of
https://github.com/mudler/LocalAI.git
synced 2026-06-27 01:47:18 -04:00
167768cac3
New backend = stock llama-cpp grpc-server + the paged patchset (forces LLAMA_PAGED=on), shipped as its own meta-backend (mirrors turboquant, simpler: no fork pin, no grpc-server patching - the paged runtime hooks already exist in grpc-server.cpp). Stock llama-cpp untouched (LLAMA_PAGED?=on retained; the de-risk flip deferred for sign-off). Gallery: qwen3.6-27b-nvfp4 (dense) + qwen3.6-35b-a3b-nvfp4 (MoE) with the benchmark run config (paged_kv, max_batch_tokens, parallel, flash_attention, f16), mudler/ GGUF uris (sha256 TODO until publish). Importer dropdown entry + tests. Assisted-by: Claude:opus-4.8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
67 lines
3.0 KiB
Bash
Executable File
67 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Script to copy the appropriate libraries based on architecture
|
|
# This script is used in the final stage of the Dockerfile
|
|
|
|
set -e
|
|
|
|
CURDIR=$(dirname "$(realpath $0)")
|
|
REPO_ROOT="${CURDIR}/../../.."
|
|
|
|
# Create lib directory
|
|
mkdir -p $CURDIR/package/lib
|
|
|
|
cp -avrf $CURDIR/llama-cpp-localai-paged-* $CURDIR/package/
|
|
cp -rfv $CURDIR/run.sh $CURDIR/package/
|
|
|
|
# Bundle the ggml shared backends from the CPU_ALL_VARIANTS build into package/lib. ggml
|
|
# discovers the per-microarch libggml-cpu-*.so by scanning the executable directory, which
|
|
# (via the bundled lib/ld.so that run.sh launches through) resolves to lib/. See the
|
|
# matching comment in backend/cpp/llama-cpp/package.sh. No-op on the fallback/ROCm builds.
|
|
if [ -d "$CURDIR/ggml-shared-libs" ]; then
|
|
echo "Bundling ggml shared backends (CPU_ALL_VARIANTS)..."
|
|
cp -avf $CURDIR/ggml-shared-libs/*.so* $CURDIR/package/lib/
|
|
fi
|
|
|
|
# Detect architecture and copy appropriate libraries
|
|
if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
|
|
# x86_64 architecture
|
|
echo "Detected x86_64 architecture, copying x86_64 libraries..."
|
|
cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
|
|
cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
|
|
cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
|
|
cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
|
|
cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
|
|
cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
|
|
cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
|
|
cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
|
|
cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
|
|
elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
|
|
# ARM64 architecture
|
|
echo "Detected ARM64 architecture, copying ARM64 libraries..."
|
|
cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
|
|
cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
|
|
cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
|
|
cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
|
|
cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
|
|
cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
|
|
cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
|
|
cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
|
|
cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
|
|
else
|
|
echo "Error: Could not detect architecture"
|
|
exit 1
|
|
fi
|
|
|
|
# Package GPU libraries based on BUILD_TYPE
|
|
GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
|
|
if [ -f "$GPU_LIB_SCRIPT" ]; then
|
|
echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
|
|
source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
|
|
package_gpu_libs
|
|
fi
|
|
|
|
echo "Packaging completed successfully"
|
|
ls -liah $CURDIR/package/
|
|
ls -liah $CURDIR/package/lib/
|