diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 705768b59..13e67c6fe 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -32,16 +32,30 @@ jobs: if: github.repository == 'mudler/LocalAI' runs-on: ubuntu-latest outputs: - matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }} matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} - merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }} - has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} - has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }} + # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to + # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). + matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }} + merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }} + has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }} + has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }} + matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }} + merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }} + has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }} + has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }} + matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }} + merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }} + has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }} + has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }} + matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }} + merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }} + has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }} + has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }} steps: - name: Checkout repository uses: actions/checkout@v7 @@ -109,9 +123,9 @@ jobs: # take their full ~6h cold without blocking manifest assembly for the # multi-arch backends whose per-arch digests would otherwise sit untagged # on quay long enough to be GC'd. - backend-jobs-singlearch: + backend-jobs-singlearch-1: needs: generate-matrix - if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true' + if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true' uses: ./.github/workflows/backend_build.yml with: tag-latest: ${{ matrix.tag-latest }} @@ -138,7 +152,100 @@ jobs: strategy: fail-fast: false max-parallel: 8 - matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }} + + backend-jobs-singlearch-2: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }} + + backend-jobs-singlearch-3: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }} + + backend-jobs-singlearch-4: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }} # Apply tags to per-arch digests via `imagetools create`. Split into two # jobs that mirror the build split so each merge waits ONLY on its @@ -174,10 +281,12 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }} - backend-merge-jobs-singlearch: - needs: [generate-matrix, backend-jobs-singlearch] - # See note on backend-merge-jobs-multiarch above for !cancelled(). - if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }} + # One merge shard per build shard: backend-merge-jobs-singlearch- needs only + # backend-jobs-singlearch-, preserving the "merge waits only on its own + # build" property while staying under the 256-jobs-per-matrix limit. + backend-merge-jobs-singlearch-1: + needs: [generate-matrix, backend-jobs-singlearch-1] + if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }} uses: ./.github/workflows/backend_merge.yml with: tag-latest: ${{ matrix.tag-latest }} @@ -189,7 +298,55 @@ jobs: quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }} + + backend-merge-jobs-singlearch-2: + needs: [generate-matrix, backend-jobs-singlearch-2] + if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }} + + backend-merge-jobs-singlearch-3: + needs: [generate-matrix, backend-jobs-singlearch-3] + if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }} + + backend-merge-jobs-singlearch-4: + needs: [generate-matrix, backend-jobs-singlearch-4] + if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }} + dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }} + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }} backend-jobs-darwin: needs: generate-matrix diff --git a/.github/workflows/backend_pr.yml b/.github/workflows/backend_pr.yml index 9517651e4..c13c444c4 100644 --- a/.github/workflows/backend_pr.yml +++ b/.github/workflows/backend_pr.yml @@ -11,16 +11,30 @@ jobs: generate-matrix: runs-on: ubuntu-latest outputs: - matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }} matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }} matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }} merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }} - merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }} - has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }} has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }} has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }} has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }} - has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }} + # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to + # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js). + matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }} + merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }} + has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }} + has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }} + matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }} + merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }} + has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }} + has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }} + matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }} + merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }} + has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }} + has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }} + matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }} + merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }} + has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }} + has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }} steps: - name: Checkout repository uses: actions/checkout@v7 @@ -71,10 +85,10 @@ jobs: fail-fast: true max-parallel: 8 matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-multiarch']) }} - backend-jobs-singlearch: + backend-jobs-singlearch-1: needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true' uses: ./.github/workflows/backend_build.yml - if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true' with: tag-latest: ${{ matrix.tag-latest }} tag-suffix: ${{ matrix.tag-suffix }} @@ -98,7 +112,94 @@ jobs: strategy: fail-fast: true max-parallel: 8 - matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }} + + backend-jobs-singlearch-2: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }} + + backend-jobs-singlearch-3: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }} + + backend-jobs-singlearch-4: + needs: generate-matrix + if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true' + uses: ./.github/workflows/backend_build.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + platform-tag: ${{ matrix.platform-tag || '' }} + runs-on: ${{ matrix.runs-on }} + builder-base-image: ${{ matrix.builder-base-image || '' }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + ubuntu-version: ${{ matrix.ubuntu-version }} + amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + max-parallel: 8 + matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }} backend-merge-jobs-multiarch: needs: [generate-matrix, backend-jobs-multiarch] # backend_merge.yml's push-side steps are all gated on @@ -118,9 +219,9 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }} - backend-merge-jobs-singlearch: - needs: [generate-matrix, backend-jobs-singlearch] - if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }} + backend-merge-jobs-singlearch-1: + needs: [generate-matrix, backend-jobs-singlearch-1] + if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }} uses: ./.github/workflows/backend_merge.yml with: tag-latest: ${{ matrix.tag-latest }} @@ -130,7 +231,49 @@ jobs: quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }} + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }} + + backend-merge-jobs-singlearch-2: + needs: [generate-matrix, backend-jobs-singlearch-2] + if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }} + + backend-merge-jobs-singlearch-3: + needs: [generate-matrix, backend-jobs-singlearch-3] + if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }} + + backend-merge-jobs-singlearch-4: + needs: [generate-matrix, backend-jobs-singlearch-4] + if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }} + uses: ./.github/workflows/backend_merge.yml + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }} backend-jobs-darwin: needs: generate-matrix uses: ./.github/workflows/backend_build_darwin.yml diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index 758cdb869..3e0355f62 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -209,23 +209,75 @@ function splitByArch(entries) { return { multiarch, singlearch }; } +// GitHub Actions refuses to instantiate a matrix with more than 256 jobs. When +// it happens the job doesn't error visibly — it hangs forever at "Waiting for +// pending jobs" and the whole run is marked `failure` while every *other* job +// stays green (seen on the v4.6.1 tag build, run 28786533892: 268 single-arch +// entries, zero single-arch jobs ever created). The single-arch list is the +// one that grows unbounded as backends are added, so we shard it across a +// fixed number of matrix jobs instead of feeding one oversized matrix. +// +// SINGLEARCH_SHARDS MUST equal the number of backend-jobs-singlearch- +// (and backend-merge-jobs-singlearch-) blocks defined in backend.yml and +// backend_pr.yml. Bump all three together. +const SINGLEARCH_SHARDS = 4; +const GHA_MATRIX_LIMIT = 256; + +// Split `arr` into exactly `shards` balanced, contiguous chunks. Earlier chunks +// absorb the remainder when the length doesn't divide evenly; trailing chunks +// may be empty when there are fewer entries than shards (those emit a +// has-backends-singlearch-=false flag so their job is skipped). +function chunkEqually(arr, shards) { + const out = []; + const base = Math.floor(arr.length / shards); + const rem = arr.length % shards; + let idx = 0; + for (let i = 0; i < shards; i++) { + const size = base + (i < rem ? 1 : 0); + out.push(arr.slice(idx, idx + size)); + idx += size; + } + return out; +} + +// Emit the sharded single-arch build + merge matrices and their has-* gates. +// Called with the full or filtered single-arch entry list. +function emitSinglearchShards(singlearch) { + const shards = chunkEqually(singlearch, SINGLEARCH_SHARDS); + for (let i = 0; i < SINGLEARCH_SHARDS; i++) { + const shard = shards[i]; + // Fail loudly rather than let GitHub silently drop the overflow: a shard at + // or above the limit means SINGLEARCH_SHARDS (and the matching job blocks in + // both workflows) need to grow. + if (shard.length >= GHA_MATRIX_LIMIT) { + throw new Error( + `single-arch shard ${i + 1} has ${shard.length} entries (>= ${GHA_MATRIX_LIMIT}, ` + + `GitHub's per-matrix job limit). Increase SINGLEARCH_SHARDS in ` + + `scripts/changed-backends.js and add matching backend-jobs-singlearch- / ` + + `backend-merge-jobs-singlearch- blocks to backend.yml and backend_pr.yml.` + ); + } + const merge = computeMergeMatrix(shard); + const n = i + 1; + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch-${n}=${shard.length > 0 ? 'true' : 'false'}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch-${n}=${merge.include.length > 0 ? 'true' : 'false'}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch-${n}=${JSON.stringify({ include: shard })}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch-${n}=${JSON.stringify(merge)}\n`); + } +} + function emitFullMatrix() { const { multiarch, singlearch } = splitByArch(includes); const mergeMatrixMultiarch = computeMergeMatrix(multiarch); - const mergeMatrixSinglearch = computeMergeMatrix(singlearch); const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false'; - const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false'; fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${singlearch.length > 0 ? 'true' : 'false'}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`); + emitSinglearchShards(singlearch); for (const backend of allBackendPaths.keys()) { fs.appendFileSync(process.env.GITHUB_OUTPUT, `${backend}=true\n`); } @@ -249,29 +301,23 @@ function emitFilteredMatrix(changedFiles) { console.log("Filtered files Darwin:", filteredDarwin); const { multiarch, singlearch } = splitByArch(filtered); - const hasBackendsSinglearch = singlearch.length > 0 ? 'true' : 'false'; const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false'; const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false'; - console.log("Has single-arch backends?:", hasBackendsSinglearch); + console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false'); console.log("Has multi-arch backends?:", hasBackendsMultiarch); console.log("Has Darwin backends?:", hasBackendsDarwin); const mergeMatrixMultiarch = computeMergeMatrix(multiarch); - const mergeMatrixSinglearch = computeMergeMatrix(singlearch); const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false'; - const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false'; fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${hasBackendsSinglearch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`); fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`); - fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`); + emitSinglearchShards(singlearch); // Per-backend boolean outputs for (const [backend, pathPrefix] of allBackendPaths) {