ci: layered Python base images for cross-matrix dedup

The 234-entry backend matrix runs the same apt-update + GPU SDK install +
Python toolchain bootstrap into N independent registry-cache tags. Factor
that shared work out into a tier-1+2 base image (lang × accel × ubuntu ×
cuda) built once per workflow run, then consumed by every backend that
matches its tuple via BASE_IMAGE_PREBUILT.

The matrix data moves to .github/backend-matrix.yaml so backend.yml can
switch to fromJSON without duplicating the matrix. scripts/changed-backends.js
reads the data file, derives the deduplicated bases-matrix, annotates each
Python entry with the right base-image-prebuilt ref, and runs a collision
check that fails loudly if a future matrix change makes two consumers want
incompatible bases under the same tag-stem.

PR builds tag with -pr<N> so end-to-end validation lives within one PR;
master builds tag without the suffix. The base-images registry cache
parallels the existing per-matrix-entry caches.

Adding a new (accel, cuda) flavour is a backend-matrix.yaml edit; adding
a new language tier is a Dockerfile.<lang> recipe + a slim of the
consumer Dockerfile (script auto-detects via .docker/bases/).

10 distinct bases derive from the current 234 entries, replacing the
inline bootstrap that previously ran into ~10 separate cache tags.

Assisted-by: Claude:opus-4-7-1m [Claude Code]
Signed-off-by: Richard Palethorpe <io@richiejp.com>

.github/workflows/backend_build.yml

@@ -63,6 +63,16 @@ on:
         required: false
         default: ''
         type: string
+      base-image-prebuilt:
+        description: |
+          Optional reference to a prebuilt accel/lang base image
+          (quay.io/go-skynet/localai-base:<tag>). When set, the backend
+          Dockerfile FROMs this image instead of running the inline
+          bootstrap. See .github/workflows/base_images_python.yml and
+          .agents/ci-caching.md.
+        required: false
+        default: ''
+        type: string
     secrets:
       dockerUsername:
         required: false
@@ -228,6 +238,7 @@ jobs:
             APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }}
             APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }}
             DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }}
+            BASE_IMAGE_PREBUILT=${{ inputs.base-image-prebuilt }}
           context: ${{ inputs.context }}
           file: ${{ inputs.dockerfile }}
           cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}
@@ -254,6 +265,7 @@ jobs:
             APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }}
             APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }}
             DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }}
+            BASE_IMAGE_PREBUILT=${{ inputs.base-image-prebuilt }}
           context: ${{ inputs.context }}
           file: ${{ inputs.dockerfile }}
           cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}

.github/workflows/backend_pr.yml

@@ -13,8 +13,10 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       matrix-darwin: ${{ steps.set-matrix.outputs.matrix-darwin }}
+      bases-matrix: ${{ steps.set-matrix.outputs.bases-matrix }}
       has-backends: ${{ steps.set-matrix.outputs.has-backends }}
       has-backends-darwin: ${{ steps.set-matrix.outputs.has-backends-darwin }}
+      has-bases: ${{ steps.set-matrix.outputs.has-bases }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -27,7 +29,8 @@ jobs:
           bun add js-yaml
           bun add @octokit/core
 
-      # filters the matrix in backend.yml
+      # Filters the matrix from backend.yml against this PR's changed files
+      # AND derives the deduplicated bases-matrix consumed by build-bases.
       - name: Filter matrix for changed backends
         id: set-matrix
         env:
@@ -35,10 +38,34 @@ jobs:
           GITHUB_EVENT_PATH: ${{ github.event_path }}
         run: bun run scripts/changed-backends.js
 
-  backend-jobs:
+  build-bases:
     needs: generate-matrix
+    if: needs.generate-matrix.outputs.has-bases == 'true'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.bases-matrix) }}
+    uses: ./.github/workflows/base_images.yml
+    with:
+      lang: ${{ matrix.lang }}
+      base-image: ${{ matrix.base-image }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      platforms: ${{ matrix.platforms }}
+      runs-on: ${{ matrix.runs-on }}
+      tag-stem: ${{ matrix.tag-stem }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+
+  backend-jobs:
+    needs: [generate-matrix, build-bases]
     uses: ./.github/workflows/backend_build.yml
-    if: needs.generate-matrix.outputs.has-backends == 'true'
+    if: |
+      always() && needs.generate-matrix.outputs.has-backends == 'true' &&
+      (needs.build-bases.result == 'success' || needs.build-bases.result == 'skipped')
     with:
       tag-latest: ${{ matrix.tag-latest }}
       tag-suffix: ${{ matrix.tag-suffix }}
@@ -54,12 +81,17 @@ jobs:
       context: ${{ matrix.context }}
       ubuntu-version: ${{ matrix.ubuntu-version }}
       amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+      # The script annotates each filtered Python entry with the prebuilt
+      # base ref it should consume; non-Python entries get '' and run their
+      # own inline bootstrap.
+      base-image-prebuilt: ${{ matrix.base-image-prebuilt || '' }}
     secrets:
       quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: true
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
+
   backend-jobs-darwin:
     needs: generate-matrix
     uses: ./.github/workflows/backend_build_darwin.yml

.github/workflows/base_images.yml

@@ -0,0 +1,145 @@
+---
+name: 'build base image (reusable)'
+
+# Builds and pushes one (lang, accel, arch, ubuntu, cuda) base image flavour
+# to quay.io/go-skynet/localai-base. Consumed by backend builds via the
+# BASE_IMAGE_PREBUILT build-arg. PR builds tag with `-pr${PR_NUMBER}` so the
+# same PR's backend matrix can opt-in to the freshly-built base; master
+# builds overwrite the unsuffixed tag for downstream consumption. See
+# .agents/ci-caching.md for the full tagging scheme.
+
+on:
+  workflow_call:
+    inputs:
+      lang:
+        description: 'Language toolchain (matches .docker/bases/Dockerfile.<lang>)'
+        required: true
+        type: string
+      base-image:
+        description: 'Upstream base image (ubuntu:24.04, rocm/dev-ubuntu-24.04:..., etc.)'
+        required: true
+        type: string
+      build-type:
+        description: 'BUILD_TYPE: empty for CPU, cublas, hipblas, vulkan, l4t, ...'
+        default: ''
+        type: string
+      cuda-major-version:
+        description: 'CUDA major version (only meaningful for cublas/l4t)'
+        default: '12'
+        type: string
+      cuda-minor-version:
+        description: 'CUDA minor version'
+        default: '9'
+        type: string
+      ubuntu-version:
+        description: 'Ubuntu version code (2204, 2404)'
+        default: '2404'
+        type: string
+      platforms:
+        description: 'Single platform per call (linux/amd64 or linux/arm64)'
+        required: true
+        type: string
+      runs-on:
+        description: 'Runner label'
+        required: true
+        type: string
+      tag-stem:
+        description: 'Stable portion of the image tag (e.g. python-cpu-amd64-2404)'
+        required: true
+        type: string
+      skip-drivers:
+        description: 'Pass-through to the base Dockerfile'
+        default: 'false'
+        type: string
+    secrets:
+      quayUsername:
+        required: false
+      quayPassword:
+        required: false
+    outputs:
+      image-ref:
+        description: 'Full image reference of the built base'
+        value: ${{ jobs.base-build.outputs.image-ref }}
+
+jobs:
+  base-build:
+    runs-on: ${{ inputs.runs-on }}
+    env:
+      quay_username: ${{ secrets.quayUsername }}
+    outputs:
+      image-ref: ${{ steps.compute_ref.outputs.ref }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Configure apt mirror on runner
+        id: apt_mirror
+        uses: ./.github/actions/configure-apt-mirror
+
+      - name: Free Disk Space (Ubuntu)
+        if: inputs.runs-on == 'ubuntu-latest'
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
+      - name: Compute image ref
+        id: compute_ref
+        run: |
+          stem='${{ inputs.tag-stem }}'
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            tag="${stem}-pr${{ github.event.number }}"
+          else
+            tag="${stem}"
+          fi
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+          echo "ref=quay.io/go-skynet/localai-base:${tag}" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+
+      - name: Login to Quay.io
+        if: ${{ env.quay_username != '' }}
+        uses: docker/login-action@v4
+        with:
+          registry: quay.io
+          username: ${{ secrets.quayUsername }}
+          password: ${{ secrets.quayPassword }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v7
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./.docker/bases/Dockerfile.${{ inputs.lang }}
+          build-args: |
+            BUILD_TYPE=${{ inputs.build-type }}
+            CUDA_MAJOR_VERSION=${{ inputs.cuda-major-version }}
+            CUDA_MINOR_VERSION=${{ inputs.cuda-minor-version }}
+            BASE_IMAGE=${{ inputs.base-image }}
+            UBUNTU_VERSION=${{ inputs.ubuntu-version }}
+            SKIP_DRIVERS=${{ inputs.skip-drivers }}
+            APT_MIRROR=${{ steps.apt_mirror.outputs.effective-mirror }}
+            APT_PORTS_MIRROR=${{ steps.apt_mirror.outputs.effective-ports-mirror }}
+          platforms: ${{ inputs.platforms }}
+          # Push on PRs as well (if creds present) so the PR's backend matrix
+          # can opt-in to the freshly-built base via -pr${N} tag.
+          push: ${{ env.quay_username != '' }}
+          tags: ${{ steps.compute_ref.outputs.ref }}
+          cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:base-${{ inputs.tag-stem }}
+          cache-to: type=registry,ref=quay.io/go-skynet/ci-cache:base-${{ inputs.tag-stem }},mode=max,ignore-error=true
+
+      - name: job summary
+        run: |
+          echo "Built base image: ${{ steps.compute_ref.outputs.ref }}" >> "$GITHUB_STEP_SUMMARY"