mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-07 23:07:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): shard single-arch backend matrix under GitHub's 256-job limit (#10703)

Browse Source 
GitHub Actions refuses to instantiate a matrix that would generate more
than 256 jobs. It does so silently: the job hangs forever at "Waiting for
pending jobs" and the whole run is marked `failure` while every other job
stays green. This is exactly what happened on the v4.6.1 tag build
(run 28786533892): the single-arch build matrix had grown to 268 entries,
so `backend-jobs-singlearch` (and its downstream merge) never produced a
single job, and the release build "failed" with no failing job to point at.

The single-arch list is the one that grows unbounded as backends are added,
so shard it across a fixed number of matrix jobs (SINGLEARCH_SHARDS=4,
~67 entries each today, headroom to ~1020 backends). Each merge shard
`needs:` only its matching build shard, preserving the "merge waits only on
its own build" property that keeps slow CUDA/ROCm builds from gating
multi-arch manifest assembly.

changed-backends.js now emits per-shard matrix/has-* outputs and throws
loudly if a shard ever reaches the 256 limit (telling the maintainer to
bump SINGLEARCH_SHARDS and add matching job blocks) instead of letting
GitHub drop the overflow silently. backend.yml and backend_pr.yml define
the four build + four merge shard jobs; multi-arch and darwin groups are
untouched.


Assisted-by: Claude:claude-opus-4-8 [Claude Code]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-07-06 21:38:12 +02:00
committed by GitHub
parent a6cf67cc6b
commit 29db4ab414
3 changed files with 383 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
scripts/changed-backends.js
View File

@@ -209,23 +209,75 @@ function splitByArch(entries) {
return { multiarch, singlearch };
}
// GitHub Actions refuses to instantiate a matrix with more than 256 jobs. When
// it happens the job doesn't error visibly — it hangs forever at "Waiting for
// pending jobs" and the whole run is marked `failure` while every *other* job
// stays green (seen on the v4.6.1 tag build, run 28786533892: 268 single-arch
// entries, zero single-arch jobs ever created). The single-arch list is the
// one that grows unbounded as backends are added, so we shard it across a
// fixed number of matrix jobs instead of feeding one oversized matrix.
//
// SINGLEARCH_SHARDS MUST equal the number of backend-jobs-singlearch-<n>
// (and backend-merge-jobs-singlearch-<n>) blocks defined in backend.yml and
// backend_pr.yml. Bump all three together.
const SINGLEARCH_SHARDS = 4;
const GHA_MATRIX_LIMIT = 256;
// Split `arr` into exactly `shards` balanced, contiguous chunks. Earlier chunks
// absorb the remainder when the length doesn't divide evenly; trailing chunks
// may be empty when there are fewer entries than shards (those emit a
// has-backends-singlearch-<n>=false flag so their job is skipped).
function chunkEqually(arr, shards) {
const out = [];
const base = Math.floor(arr.length / shards);
const rem = arr.length % shards;
let idx = 0;
for (let i = 0; i < shards; i++) {
const size = base + (i < rem ? 1 : 0);
out.push(arr.slice(idx, idx + size));
idx += size;
}
return out;
}
// Emit the sharded single-arch build + merge matrices and their has-* gates.
// Called with the full or filtered single-arch entry list.
function emitSinglearchShards(singlearch) {
const shards = chunkEqually(singlearch, SINGLEARCH_SHARDS);
for (let i = 0; i < SINGLEARCH_SHARDS; i++) {
const shard = shards[i];
// Fail loudly rather than let GitHub silently drop the overflow: a shard at
// or above the limit means SINGLEARCH_SHARDS (and the matching job blocks in
// both workflows) need to grow.
if (shard.length >= GHA_MATRIX_LIMIT) {
throw new Error(
`single-arch shard ${i + 1} has ${shard.length} entries (>= ${GHA_MATRIX_LIMIT}, ` +
`GitHub's per-matrix job limit). Increase SINGLEARCH_SHARDS in ` +
`scripts/changed-backends.js and add matching backend-jobs-singlearch-<n> / ` +
`backend-merge-jobs-singlearch-<n> blocks to backend.yml and backend_pr.yml.`
);
}
const merge = computeMergeMatrix(shard);
const n = i + 1;
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch-${n}=${shard.length > 0 ? 'true' : 'false'}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch-${n}=${merge.include.length > 0 ? 'true' : 'false'}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch-${n}=${JSON.stringify({ include: shard })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch-${n}=${JSON.stringify(merge)}\n`);
}
}
function emitFullMatrix() {
const { multiarch, singlearch } = splitByArch(includes);
const mergeMatrixMultiarch = computeMergeMatrix(multiarch);
const mergeMatrixSinglearch = computeMergeMatrix(singlearch);
const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false';
const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false';
fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=true\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${singlearch.length > 0 ? 'true' : 'false'}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${multiarch.length > 0 ? 'true' : 'false'}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=true\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: includesDarwin })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`);
emitSinglearchShards(singlearch);
for (const backend of allBackendPaths.keys()) {
fs.appendFileSync(process.env.GITHUB_OUTPUT, `${backend}=true\n`);
}
@@ -249,29 +301,23 @@ function emitFilteredMatrix(changedFiles) {
console.log("Filtered files Darwin:", filteredDarwin);
const { multiarch, singlearch } = splitByArch(filtered);
const hasBackendsSinglearch = singlearch.length > 0 ? 'true' : 'false';
const hasBackendsMultiarch = multiarch.length > 0 ? 'true' : 'false';
const hasBackendsDarwin = filteredDarwin.length > 0 ? 'true' : 'false';
console.log("Has single-arch backends?:", hasBackendsSinglearch);
console.log("Has single-arch backends?:", singlearch.length > 0 ? 'true' : 'false');
console.log("Has multi-arch backends?:", hasBackendsMultiarch);
console.log("Has Darwin backends?:", hasBackendsDarwin);
const mergeMatrixMultiarch = computeMergeMatrix(multiarch);
const mergeMatrixSinglearch = computeMergeMatrix(singlearch);
const hasMergesMultiarch = mergeMatrixMultiarch.include.length > 0 ? 'true' : 'false';
const hasMergesSinglearch = mergeMatrixSinglearch.include.length > 0 ? 'true' : 'false';
fs.appendFileSync(process.env.GITHUB_OUTPUT, `run-all=false\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-singlearch=${hasBackendsSinglearch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-multiarch=${hasBackendsMultiarch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends-darwin=${hasBackendsDarwin}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-multiarch=${hasMergesMultiarch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-merges-singlearch=${hasMergesSinglearch}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-singlearch=${JSON.stringify({ include: singlearch })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-multiarch=${JSON.stringify({ include: multiarch })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix-darwin=${JSON.stringify({ include: filteredDarwin })}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-multiarch=${JSON.stringify(mergeMatrixMultiarch)}\n`);
fs.appendFileSync(process.env.GITHUB_OUTPUT, `merge-matrix-singlearch=${JSON.stringify(mergeMatrixSinglearch)}\n`);
emitSinglearchShards(singlearch);
// Per-backend boolean outputs
for (const [backend, pathPrefix] of allBackendPaths) {