From a4e730979dec876eb6913445a6a568797dc47731 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 27 Jun 2026 12:29:15 +0000 Subject: [PATCH] feat(paged): restrict llama-cpp-localai-paged to CUDA-only build targets The paged backend previously built for cublas/cuda, cpu, vulkan, sycl, hipblas and darwin/metal. On non-CUDA the patchset's wins are inert: the GDN fusions are gated off (patch 0030) and NVFP4 falls back to dequant, so the backend is neutral-to-negative there (README section 4c). The darwin grpc-server link also fails on undefined upstream server symbols, turning CI red. Both broken and pointless off-CUDA, so ship CUDA-only. - backend-matrix.yml: drop the hipblas, sycl f32/f16, cpu amd64/arm64, vulkan amd64/arm64 and metal-darwin rows for this backend; keep the four cublas rows (cuda-12, cuda-13, nvidia-l4t cuda-12 and cuda-13). - index.yaml: meta-backend (and -development) capabilities are now CUDA-only with default pointing at cuda12 (mirrors faster-qwen3-tts); removed the orphaned cpu/rocm/sycl/vulkan/metal variant entries. - Removed the now-unused darwin build script and its Makefile target / .NOTPARALLEL entry / backend_build_darwin.yml step. - Documented the CUDA-only build coverage in the patch README and plan. Non-CUDA users should use the stock llama-cpp backend. Assisted-by: Claude:opus-4.8 [Claude Code] Signed-off-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 112 ------------------ .github/workflows/backend_build_darwin.yml | 12 +- Makefile | 9 +- .../paged/LOCALAI_LLAMACPP_BACKEND_PLAN.md | 7 ++ .../patches/paged/README.md | 8 ++ backend/index.yaml | 81 ++----------- .../build/llama-cpp-localai-paged-darwin.sh | 95 --------------- 7 files changed, 25 insertions(+), 299 deletions(-) delete mode 100755 scripts/build/llama-cpp-localai-paged-darwin.sh diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 40c5855dd..a616b48f8 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -4928,78 +4928,6 @@ include: backend: "llama-cpp-localai-paged" dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" context: "./" - - build-type: 'hipblas' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/amd64' - tag-latest: 'auto' - tag-suffix: '-gpu-rocm-hipblas-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' - runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - - build-type: 'sycl_f32' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/amd64' - tag-latest: 'auto' - tag-suffix: '-gpu-intel-sycl-f32-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-intel-amd64' - runs-on: 'ubuntu-latest' - base-image: "intel/oneapi-basekit:2025.3.2-0-devel-ubuntu24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - - build-type: 'sycl_f16' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/amd64' - tag-latest: 'auto' - tag-suffix: '-gpu-intel-sycl-f16-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-intel-amd64' - runs-on: 'ubuntu-latest' - base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - - build-type: '' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/amd64' - platform-tag: 'amd64' - tag-latest: 'auto' - tag-suffix: '-cpu-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-amd64' - runs-on: 'ubuntu-latest' - base-image: "ubuntu:24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - - build-type: '' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/arm64' - platform-tag: 'arm64' - tag-latest: 'auto' - tag-suffix: '-cpu-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-arm64' - runs-on: 'ubuntu-24.04-arm' - base-image: "ubuntu:24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -5014,36 +4942,6 @@ include: dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" context: "./" ubuntu-version: '2204' - - build-type: 'vulkan' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/amd64' - platform-tag: 'amd64' - tag-latest: 'auto' - tag-suffix: '-gpu-vulkan-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-amd64' - runs-on: 'ubuntu-latest' - base-image: "ubuntu:24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' - - build-type: 'vulkan' - cuda-major-version: "" - cuda-minor-version: "" - platforms: 'linux/arm64' - platform-tag: 'arm64' - tag-latest: 'auto' - tag-suffix: '-gpu-vulkan-llama-cpp-localai-paged' - builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-arm64' - runs-on: 'ubuntu-24.04-arm' - base-image: "ubuntu:24.04" - skip-drivers: 'false' - backend: "llama-cpp-localai-paged" - dockerfile: "./backend/Dockerfile.llama-cpp-localai-paged" - context: "./" - ubuntu-version: '2404' # Darwin matrix (consumed by backend-jobs-darwin). includeDarwin: @@ -5071,16 +4969,6 @@ includeDarwin: - backend: "llama-cpp" tag-suffix: "-metal-darwin-arm64-llama-cpp" lang: "go" - # llama-cpp-localai-paged on Darwin: same bespoke CPU_ALL_VARIANTS + Metal build - # as stock llama-cpp (driven by make backends/llama-cpp-localai-paged-darwin), - # reusing backend/cpp/llama-cpp sources, with the paged patch series applied by the wrapper. lang=go selects the - # runner/toolchain only; the source path is C++. Metal delivers paged-KV (the - # NVFP4 FP4-MMA fast path is CUDA/Blackwell-only) and the GDN/conv fused ops have - # no Metal kernel, so a gated-DeltaNet (qwen35) model falls back to the CPU - # reference op at runtime (made safe by the fused-op backend gate, patch 0030). - - backend: "llama-cpp-localai-paged" - tag-suffix: "-metal-darwin-arm64-llama-cpp-localai-paged" - lang: "go" - backend: "stablediffusion-ggml" tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml" build-type: "metal" diff --git a/.github/workflows/backend_build_darwin.yml b/.github/workflows/backend_build_darwin.yml index 2af407cab..36b8b393f 100644 --- a/.github/workflows/backend_build_darwin.yml +++ b/.github/workflows/backend_build_darwin.yml @@ -230,16 +230,6 @@ jobs: make protogen-go make backends/llama-cpp-darwin - # llama-cpp-localai-paged reuses the same bespoke llama-cpp darwin build path - # (CPU_ALL_VARIANTS + Metal + otool dylib bundling) via its own wrapper script, - # so it gets a dedicated step like stock llama-cpp rather than the generic - # build-darwin-go-backend mold. - - name: Build ${{ inputs.backend }}-darwin (llama-cpp-localai-paged) - if: inputs.backend == 'llama-cpp-localai-paged' - run: | - make protogen-go - make backends/llama-cpp-localai-paged-darwin - - name: Build ds4 backend (Darwin Metal) if: inputs.backend == 'ds4' run: | @@ -255,7 +245,7 @@ jobs: make backends/privacy-filter-darwin - name: Build ${{ inputs.backend }}-darwin - if: inputs.backend != 'llama-cpp' && inputs.backend != 'llama-cpp-localai-paged' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' + if: inputs.backend != 'llama-cpp' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' run: | make protogen-go BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend diff --git a/Makefile b/Makefile index e6a073bf1..0bf77a668 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/llama-cpp-localai-paged backends/llama-cpp-localai-paged-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/llama-cpp-localai-paged GOCMD=go GOTEST=$(GOCMD) test @@ -1141,13 +1141,6 @@ backends/llama-cpp-darwin: build bash ./scripts/build/llama-cpp-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)" -# llama-cpp-localai-paged on Darwin: same bespoke CPU_ALL_VARIANTS + Metal build as -# stock llama-cpp (otool dylib bundling), driven through the paged wrapper Makefile, -# which applies its own vendored paged patch series. Mirrors backends/llama-cpp-darwin. -backends/llama-cpp-localai-paged-darwin: build - bash ./scripts/build/llama-cpp-localai-paged-darwin.sh - ./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp-localai-paged.tar)" - backends/ds4-darwin: build bash ./scripts/build/ds4-darwin.sh ./local-ai backends install "ocifile://$(abspath ./backend-images/ds4.tar)" diff --git a/backend/cpp/llama-cpp-localai-paged/patches/paged/LOCALAI_LLAMACPP_BACKEND_PLAN.md b/backend/cpp/llama-cpp-localai-paged/patches/paged/LOCALAI_LLAMACPP_BACKEND_PLAN.md index 3075f921a..3a2f06e62 100644 --- a/backend/cpp/llama-cpp-localai-paged/patches/paged/LOCALAI_LLAMACPP_BACKEND_PLAN.md +++ b/backend/cpp/llama-cpp-localai-paged/patches/paged/LOCALAI_LLAMACPP_BACKEND_PLAN.md @@ -3,6 +3,13 @@ Scoping deliverable only. NOTHING is changed by this document. It is grounded in the actual repo structure (read 2026-06-26 in worktree feat+paged-attention), not assumptions. +SHIPPED REALITY (update 2026-06-27): the backend ships CUDA-only. The matrix rows and +the index.yaml meta-backend keep ONLY the CUDA/cublas variants (cuda-12, cuda-13, and +the nvidia-l4t arm64 cuda-12/cuda-13 Jetson rows). The cpu / vulkan / sycl / hipblas / +metal-darwin variants discussed below as optional/phase-2 were NOT shipped (and the +darwin row was removed): off-CUDA the patchset's wins gate off, so it is neutral-to- +negative there and non-CUDA users should use the stock llama-cpp backend (README 4c). + ================================================================================ 0. GROUND TRUTH (what the repo actually does today) ================================================================================ diff --git a/backend/cpp/llama-cpp-localai-paged/patches/paged/README.md b/backend/cpp/llama-cpp-localai-paged/patches/paged/README.md index aea38ffc0..bc42b0450 100644 --- a/backend/cpp/llama-cpp-localai-paged/patches/paged/README.md +++ b/backend/cpp/llama-cpp-localai-paged/patches/paged/README.md @@ -344,6 +344,14 @@ in a recommended/gallery config. ## 8. Models +> **Build coverage: CUDA-only.** This backend ships only the CUDA/cublas build +> targets (cuda-12, cuda-13, and the nvidia-l4t arm64 cuda-12/cuda-13 Jetson +> rows). There are no cpu / vulkan / sycl / hipblas / metal-darwin builds: the +> patchset's wins are CUDA/Blackwell-specific (section 4c), so off-CUDA the +> backend is neutral-to-negative and non-CUDA users should run the stock +> `llama-cpp` backend instead. The `backend/index.yaml` meta-backend resolves +> `default`/`nvidia` to a CUDA variant accordingly. + The benchmarked NVFP4 GGUFs are published and wired into the LocalAI gallery: | Gallery entry | Weights (HuggingFace) | Notes | diff --git a/backend/index.yaml b/backend/index.yaml index de1667d45..4970b9907 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -92,20 +92,18 @@ tags: - text-to-text - LLM - - CPU - GPU - - Metal - CUDA - - HIP - paged-attention - nvfp4 + # CUDA-only: the paged patchset's wins (GDN fusions, NVFP4 FP4-MMA) are + # CUDA/Blackwell-specific; off-CUDA they gate off and the backend is + # neutral-to-negative, so non-CUDA users should use the stock llama-cpp + # backend. default points at cuda12 (mirrors faster-qwen3-tts) so the gallery + # entries always resolve to a CUDA variant. capabilities: - default: "cpu-llama-cpp-localai-paged" + default: "cuda12-llama-cpp-localai-paged" nvidia: "cuda12-llama-cpp-localai-paged" - intel: "intel-sycl-f16-llama-cpp-localai-paged" - amd: "rocm-llama-cpp-localai-paged" - metal: "metal-llama-cpp-localai-paged" - vulkan: "vulkan-llama-cpp-localai-paged" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp-localai-paged" nvidia-cuda-13: "cuda13-llama-cpp-localai-paged" nvidia-cuda-12: "cuda12-llama-cpp-localai-paged" @@ -1681,11 +1679,8 @@ - !!merge <<: *llamacpplocalaipaged name: "llama-cpp-localai-paged-development" capabilities: - default: "cpu-llama-cpp-localai-paged-development" + default: "cuda12-llama-cpp-localai-paged-development" nvidia: "cuda12-llama-cpp-localai-paged-development" - intel: "intel-sycl-f16-llama-cpp-localai-paged-development" - amd: "rocm-llama-cpp-localai-paged-development" - vulkan: "vulkan-llama-cpp-localai-paged-development" nvidia-l4t: "nvidia-l4t-arm64-llama-cpp-localai-paged-development" nvidia-cuda-13: "cuda13-llama-cpp-localai-paged-development" nvidia-cuda-12: "cuda12-llama-cpp-localai-paged-development" @@ -2359,17 +2354,7 @@ uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant" mirrors: - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-turboquant -## llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "cpu-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-cpu-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "cpu-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-cpu-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-cpu-llama-cpp-localai-paged +## llama-cpp-localai-paged (CUDA-only; see backend/cpp/llama-cpp-localai-paged/patches/paged/README.md section 4c) - !!merge <<: *llamacpplocalaipaged name: "cuda12-llama-cpp-localai-paged" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp-localai-paged" @@ -2390,56 +2375,6 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-llama-cpp-localai-paged" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "rocm-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-gpu-rocm-hipblas-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "rocm-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-gpu-rocm-hipblas-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "intel-sycl-f32-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-gpu-intel-sycl-f32-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "intel-sycl-f32-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-gpu-intel-sycl-f32-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "intel-sycl-f16-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-gpu-intel-sycl-f16-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "intel-sycl-f16-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-gpu-intel-sycl-f16-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "vulkan-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-gpu-vulkan-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "vulkan-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-gpu-vulkan-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "metal-llama-cpp-localai-paged" - uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:latest-metal-darwin-arm64-llama-cpp-localai-paged -- !!merge <<: *llamacpplocalaipaged - name: "metal-llama-cpp-localai-paged-development" - uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-llama-cpp-localai-paged" - mirrors: - - localai/localai-backends:master-metal-darwin-arm64-llama-cpp-localai-paged - !!merge <<: *llamacpplocalaipaged name: "nvidia-l4t-arm64-llama-cpp-localai-paged" uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-llama-cpp-localai-paged" diff --git a/scripts/build/llama-cpp-localai-paged-darwin.sh b/scripts/build/llama-cpp-localai-paged-darwin.sh deleted file mode 100755 index 6a5779079..000000000 --- a/scripts/build/llama-cpp-localai-paged-darwin.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -set -ex - -# Darwin/Metal build for the llama-cpp-localai-paged backend. Mirrors -# scripts/build/llama-cpp-darwin.sh exactly, swapping the build dir, binary names, -# shared-lib dir and output tar for the paged wrapper. The paged wrapper Makefile -# (backend/cpp/llama-cpp-localai-paged) reuses backend/cpp/llama-cpp's CMakeLists -# /grpc-server and applies its own vendored paged patch series (patches/paged/) -# onto the cloned tree, so the Darwin/Metal path is identical: ggml -# CPU_ALL_VARIANTS + GGML_METAL=ON, and --target ggml pulls in ggml-metal via -# add_dependencies so the Metal GPU backend is produced as a loadable -# libggml-metal.dylib. The new paged GDN/conv ops have no Metal kernel, so a -# gated-DeltaNet (qwen35) model falls back to the CPU reference op at runtime -# (assert/fall-back is made SAFE by the fused-op backend gate, patch 0030); a -# non-qwen35 model gets the full paged-KV path on Metal. - -IMAGE_NAME="${IMAGE_NAME:-localai/llama-cpp-localai-paged-darwin}" - -pushd backend/cpp/llama-cpp-localai-paged - -# Single build via ggml CPU_ALL_VARIANTS: one binary plus the per-microarch Apple/arm -# dylibs (apple_m1/m2_m3/m4, armv8.x) that ggml selects at runtime. GGML_METAL stays ON -# and --target ggml also builds ggml-metal (via add_dependencies), so the Metal GPU -# backend is still produced as a loadable libggml-metal.dylib. -make llama-cpp-localai-paged-cpu-all && \ -make llama-cpp-localai-paged-grpc && \ -make llama-cpp-localai-paged-rpc-server - -popd - -mkdir -p build/darwin -mkdir -p backend-images -mkdir -p build/darwin/lib - -cp -rf backend/cpp/llama-cpp-localai-paged/llama-cpp-localai-paged-cpu-all build/darwin/ -cp -rf backend/cpp/llama-cpp-localai-paged/llama-cpp-localai-paged-grpc build/darwin/ -cp -rf backend/cpp/llama-cpp-localai-paged/llama-cpp-localai-paged-rpc-server build/darwin/ - -# Distribute the shared ggml/llama libraries from the CPU_ALL_VARIANTS build. Unlike the -# old fully-static fallback build, these have @rpath install names, so the otool loop below -# (which only copies deps that exist on disk) will not pick them up. The split is by suffix: -# - ggml emits its loadable backends (per-microarch CPU variants, metal, blas) with a .so -# suffix EVEN ON DARWIN. These go in the package ROOT next to the binary, because darwin -# run.sh execs the binary directly (no bundled ld.so) so ggml's executable-directory -# scan looks there. -# - the core libraries (libggml-base/libggml/libllama/libllama-common/libmtmd) use the -# platform .dylib suffix and are NEEDED deps; they go in lib/, resolved at load time via -# the DYLD_LIBRARY_PATH=lib that run.sh exports. -a preserves the version symlinks. -SHLIBS=backend/cpp/llama-cpp-localai-paged/ggml-shared-libs -cp -a $SHLIBS/*.so build/darwin/ -cp -a $SHLIBS/*.dylib build/darwin/lib/ - -# Set default additional libs only for Darwin on M chips (arm64) -if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then - ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-$(ls /opt/homebrew/Cellar/protobuf/**/lib/libutf8_validity*.dylib 2>/dev/null)} -else - ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-""} -fi - -for file in $ADDITIONAL_LIBS; do - cp -rfv $file build/darwin/lib -done - -for file in build/darwin/*; do - LIBS="$(otool -L $file | awk 'NR > 1 { system("echo " $1) } ' | xargs echo)" - for lib in $LIBS; do - # only libraries ending in dylib - if [[ "$lib" == *.dylib ]]; then - if [ -e "$lib" ]; then - cp -rvf "$lib" build/darwin/lib - fi - fi - done -done - -echo "--------------------------------" -echo "ADDITIONAL_LIBS: $ADDITIONAL_LIBS" -echo "--------------------------------" - -echo "Bundled libraries:" -ls -la build/darwin/lib - - -cp -rf backend/cpp/llama-cpp-localai-paged/run.sh build/darwin/ - -PLATFORMARCH="${PLATFORMARCH:-darwin/arm64}" - -./local-ai util create-oci-image \ - build/darwin/. \ - --output ./backend-images/llama-cpp-localai-paged.tar \ - --image-name $IMAGE_NAME \ - --platform $PLATFORMARCH - -rm -rf build/darwin