diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 17cc6001d..74dd9bb91 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -195,6 +195,24 @@ concurrency: - **PR events** group by PR number → newer pushes to the same PR cancel old runs (intended). - **Push events** group by `github.sha` → each master commit gets its own run; rapid-fire merges don't cancel each other (this was a real issue prior — two master pushes 11 seconds apart would cancel the first's CI). +### Consequence: builds finish out of commit order + +Because no master run supersedes another, and because the backend queue routinely runs hours deep (measured 259.7 min average queue wait against 18.6 min average execution), **completion order does not track commit order**. A build of an older commit can finish long after a newer one. + +That used to move mutable tags backwards. On 19 Jul 2026 a build of commit `10211948b` (pushed 06:45 UTC) finished at 15:40 UTC and overwrote `master-nvidia-l4t-cuda-13-arm64-longcat-video`, which a 10:41 UTC build of `626ae4d51` had already advanced to a commit containing a merged cuDNN packaging fix. Everyone pulling `master-*` got the pre-fix image for two days, hitting `CUDNN_STATUS_SUBLIBRARY_VERSION_MISMATCH` at inference. + +## Mutable tag ordering guard + +`scripts/tag-guard.mjs` (logic in `scripts/lib/tag-guard.mjs`, tests in `scripts/lib/tag-guard_test.mjs`, run by `make test-ci-scripts`) refuses to advance a mutable tag unless the incoming commit descends from the one the tag currently points at. + +- **Tag classes**: `sha--` is immutable and *always* publishes: it is how a human pins a known-good artifact, and it is how the incident above was worked around. `master-*`, `latest-*` and `v-*` are mutable and go through the guard. +- **How it decides**: reads `org.opencontainers.image.revision` off whatever the mutable tag currently points at (docker/metadata-action already stamps this at build time on the Linux path), then asks the GitHub compare API how the incoming commit relates to it. `ahead`/`identical` publish; `behind`/`diverged` are withheld. +- **Why the compare API and not `git merge-base --is-ancestor`**: the merge and darwin-publish jobs use shallow sparse checkouts. Fetching full history into each of the ~200 merge jobs per push to answer one ancestry question is not worth it. +- **Fail-open, loudly**: a tag that doesn't exist yet, an image with no revision label, an unreachable commit, or a registry/API error all publish anyway with a `::warning::` annotation and a job-summary line. A guard that failed closed on an API blip would itself stop shipping merged fixes. Every non-trivial decision is annotated; silence is the bug being fixed. +- **Where it runs**: `backend_merge.yml` (both the quay and Docker Hub `imagetools create` steps, which are the multi-arch manifest merge *and* the tagging step for single-arch backends, since `backend_build.yml` pushes by digest only) and `backend_build_darwin.yml`'s publish job. +- **Darwin caveat**: darwin images are `crane push`ed from a raw OCI tarball and carried no labels at all, so the publish job now stamps the revision with `crane mutate` after pushing. Until each darwin tag has gone through that step once, the guard has nothing to compare and fails open with a warning. +- **Release tags**: `v*` tags go through the guard too. A fresh version tag has never been published, so it hits the "tag does not exist yet" path and always publishes. The guard costs nothing there, and it does protect `latest-*` if an old release build is ever re-run. + ## Self-warming, no separate populator There is no cron job that pre-warms the BuildKit cache for individual backends. The production builds *are* the populators. The first master build of a given matrix entry pays the cold cost; subsequent same-entry master builds reuse everything that hasn't changed (apt installs, gRPC compile in the variant `builder-fromsource` stage or skipped entirely when consuming `base-grpc-*`, Python wheel installs, etc.). The base-images workflow's weekly cron is the closest thing to a populator and only refreshes the prebuilt builder bases. diff --git a/.github/workflows/backend_build_darwin.yml b/.github/workflows/backend_build_darwin.yml index e4eef7124..df028bd9c 100644 --- a/.github/workflows/backend_build_darwin.yml +++ b/.github/workflows/backend_build_darwin.yml @@ -281,6 +281,15 @@ jobs: if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: + # Sparse checkout: the publish job needs `scripts/` for the mutable-tag + # ordering guard, nothing else from the source tree. + - name: Checkout (scripts only) + uses: actions/checkout@v7 + with: + sparse-checkout: | + scripts + sparse-checkout-cone-mode: false + - name: Download ${{ inputs.backend }}.tar uses: actions/download-artifact@v8 with: @@ -328,14 +337,47 @@ jobs: latest=auto suffix=${{ inputs.tag-suffix }},onlatest=true + # Mutable tags (master-*, latest-*, v*-*) must never move backwards: a + # straggler build of an older commit finishing after a newer one would + # silently un-ship whatever the newer one fixed. See + # scripts/lib/tag-guard.mjs. Immutable sha-* tags always publish. + # + # Darwin images are pushed as a raw OCI tarball, so unlike the Linux path + # (where docker/metadata-action bakes its labels in at build time) they + # carry no org.opencontainers.image.revision label for the guard to read. + # `crane mutate` stamps it after the push. Until each tag has been + # republished once through this step the guard has nothing to compare and + # fails open with a warning, which is the pre-existing behaviour. + # + # DOCKER_METADATA_OUTPUT_JSON is set per step rather than inherited: this + # job runs docker/metadata-action twice, so the ambient env var only ever + # holds the last one's output. - name: Push Docker image (DockerHub) + env: + REGISTRY_PREFIX: 'localai/' + GITHUB_TOKEN: ${{ github.token }} + DOCKER_METADATA_OUTPUT_JSON: ${{ steps.meta.outputs.json }} run: | - for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do - crane push ${{ inputs.backend }}.tar $tag - done + set -euo pipefail + node scripts/tag-guard.mjs > "${RUNNER_TEMP}/allowed-hub-tags.txt" + while read -r tag; do + [ -n "$tag" ] || continue + crane push "${{ inputs.backend }}.tar" "$tag" + crane mutate "$tag" -t "$tag" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" + done < "${RUNNER_TEMP}/allowed-hub-tags.txt" - name: Push Docker image (Quay) + env: + REGISTRY_PREFIX: 'quay.io/' + GITHUB_TOKEN: ${{ github.token }} + DOCKER_METADATA_OUTPUT_JSON: ${{ steps.quaymeta.outputs.json }} run: | - for tag in $(echo "${{ steps.quaymeta.outputs.tags }}" | tr ',' '\n'); do - crane push ${{ inputs.backend }}.tar $tag - done + set -euo pipefail + node scripts/tag-guard.mjs > "${RUNNER_TEMP}/allowed-quay-tags.txt" + while read -r tag; do + [ -n "$tag" ] || continue + crane push "${{ inputs.backend }}.tar" "$tag" + crane mutate "$tag" -t "$tag" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" + done < "${RUNNER_TEMP}/allowed-quay-tags.txt" diff --git a/.github/workflows/backend_merge.yml b/.github/workflows/backend_merge.yml index 37f606aa9..971b81109 100644 --- a/.github/workflows/backend_merge.yml +++ b/.github/workflows/backend_merge.yml @@ -47,12 +47,14 @@ jobs: COSIGN_EXPERIMENTAL: '1' steps: # Sparse checkout: the merge job needs `.github/scripts/` (for the - # keepalive cleanup script) but none of the source tree. - - name: Checkout (.github/scripts only) + # keepalive cleanup script) and `scripts/` (for the mutable-tag ordering + # guard) but none of the source tree. + - name: Checkout (scripts only) uses: actions/checkout@v7 with: sparse-checkout: | .github/scripts + scripts sparse-checkout-cone-mode: false # `--` separator anchors the glob so we don't over-match sibling @@ -129,30 +131,44 @@ jobs: # manifest list with the user-facing tags. The resulting manifest # list is fully self-contained in local-ai-backends — child digests # only, no embedded references to ci-cache. + # + # Mutable tags (master-*, latest-*, v*-*) must never move backwards. + # Backend CI queues run hours deep and master pushes get a concurrency + # group keyed by github.sha, so a straggler build of an older commit can + # finish after a newer one and silently un-ship a merged fix (measured on + # 19 Jul 2026, see scripts/lib/tag-guard.mjs). The guard drops any mutable + # tag whose published image was built from a commit this one does not + # descend from; immutable sha-* tags always survive it. - name: Create manifest list and push (quay) if: github.event_name != 'pull_request' working-directory: /tmp/digests + env: + REGISTRY_PREFIX: 'quay.io/' + GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail - tags=$(jq -cr ' - .tags - | map(select(startswith("quay.io/"))) - | map("-t " + .) - | join(" ") - ' <<< "$DOCKER_METADATA_OUTPUT_JSON") - if [ -z "$tags" ]; then - echo "No quay.io tags from docker/metadata-action; skipping quay merge" + # Via a file, not `mapfile < <(...)`: process substitution hides the + # exit status, so a crashing guard would look like "no tags" and + # skip the merge silently. + node "$GITHUB_WORKSPACE/scripts/tag-guard.mjs" > "${RUNNER_TEMP}/allowed-quay-tags.txt" + mapfile -t allowed < "${RUNNER_TEMP}/allowed-quay-tags.txt" + if [ "${#allowed[@]}" -eq 0 ] || [ -z "${allowed[0]}" ]; then + echo "No publishable quay.io tags; skipping quay merge" exit 0 fi - # shellcheck disable=SC2086 - docker buildx imagetools create $tags \ + tags=() + for t in "${allowed[@]}"; do + tags+=(-t "$t") + done + # shellcheck disable=SC2046 + docker buildx imagetools create "${tags[@]}" \ $(printf 'quay.io/go-skynet/ci-cache@sha256:%s ' *) # Resolve the manifest-list digest (any tag points at it) so # cosign can sign by digest. Signing by tag would leave the - # signature orphaned the next time the tag moves. - first_tag=$(jq -cr ' - .tags | map(select(startswith("quay.io/"))) | .[0] - ' <<< "$DOCKER_METADATA_OUTPUT_JSON") + # signature orphaned the next time the tag moves. The guard emits + # immutable sha-* tags first, so this is the per-commit tag whenever + # one exists. + first_tag="${allowed[0]}" digest=$(docker buildx imagetools inspect "$first_tag" --format '{{.Manifest.Digest}}') # --recursive walks the list and signs every per-arch entry # too — clients that resolve a tag to a platform-specific @@ -165,24 +181,25 @@ jobs: - name: Create manifest list and push (dockerhub) if: github.event_name != 'pull_request' working-directory: /tmp/digests + env: + REGISTRY_PREFIX: 'localai/' + GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail - tags=$(jq -cr ' - .tags - | map(select(startswith("localai/"))) - | map("-t " + .) - | join(" ") - ' <<< "$DOCKER_METADATA_OUTPUT_JSON") - if [ -z "$tags" ]; then - echo "No dockerhub tags from docker/metadata-action; skipping dockerhub merge" + node "$GITHUB_WORKSPACE/scripts/tag-guard.mjs" > "${RUNNER_TEMP}/allowed-hub-tags.txt" + mapfile -t allowed < "${RUNNER_TEMP}/allowed-hub-tags.txt" + if [ "${#allowed[@]}" -eq 0 ] || [ -z "${allowed[0]}" ]; then + echo "No publishable dockerhub tags; skipping dockerhub merge" exit 0 fi - # shellcheck disable=SC2086 - docker buildx imagetools create $tags \ + tags=() + for t in "${allowed[@]}"; do + tags+=(-t "$t") + done + # shellcheck disable=SC2046 + docker buildx imagetools create "${tags[@]}" \ $(printf 'localai/localai-backends@sha256:%s ' *) - first_tag=$(jq -cr ' - .tags | map(select(startswith("localai/"))) | .[0] - ' <<< "$DOCKER_METADATA_OUTPUT_JSON") + first_tag="${allowed[0]}" digest=$(docker buildx imagetools inspect "$first_tag" --format '{{.Manifest.Digest}}') cosign sign --yes --recursive \ --registry-referrers-mode=oci-1-1 \ diff --git a/scripts/lib/tag-guard.mjs b/scripts/lib/tag-guard.mjs new file mode 100644 index 000000000..116a5aff8 --- /dev/null +++ b/scripts/lib/tag-guard.mjs @@ -0,0 +1,330 @@ +// Ordering guard for mutable backend image tags. +// +// Backend images are published to quay.io/go-skynet/local-ai-backends and +// localai/localai-backends under two kinds of tag: +// +// immutable sha-- one per commit, never reused +// mutable master- moved on every master build +// latest- +// v- +// +// Nothing used to check whether an incoming build was newer than whatever a +// mutable tag already pointed at, so the last writer won. That is not a +// theoretical race: backend CI queues run hours deep (measured ~260 min +// average queue wait against ~19 min average execution) and master pushes get +// a concurrency group keyed by github.sha, so no run supersedes another. +// Completion order does not track commit order. On 19 Jul 2026 a build of a +// commit from 06:45 UTC finished at 15:40 UTC and moved +// master-nvidia-l4t-cuda-13-arm64-longcat-video back onto a pre-fix image, +// silently un-shipping a merged cuDNN packaging fix for two days. +// +// This module decides, per tag, whether a push may proceed. Everything here is +// dependency-free; the registry and GitHub lookups take an injected fetch so +// the decision logic is unit-testable without network access +// (scripts/lib/tag-guard_test.mjs, run by `make test-ci-scripts`). + +// docker/metadata-action's `type=sha` emits `sha-<7+ hex>`, and our flavor +// appends the backend tag-suffix. Anything matching this is a per-commit +// artifact: it is how a human pins a known-good build, so it must always push. +const IMMUTABLE_TAG_RE = /^sha-[0-9a-f]{7,40}(?:-|$)/; + +const REVISION_LABEL = "org.opencontainers.image.revision"; + +const MANIFEST_ACCEPT = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(","); + +// Split "quay.io/go-skynet/local-ai-backends:master-foo" into its parts. +// Docker Hub refs carry no registry host ("localai/localai-backends:tag"), so +// a first segment without a dot or colon means Docker Hub. +export function parseImageRef(ref) { + const lastColon = ref.lastIndexOf(":"); + const lastSlash = ref.lastIndexOf("/"); + if (lastColon === -1 || lastColon < lastSlash) { + throw new Error(`image ref has no tag: ${ref}`); + } + const name = ref.slice(0, lastColon); + const tag = ref.slice(lastColon + 1); + + const segments = name.split("/"); + let host = "docker.io"; + let repository = name; + if (segments.length > 1 && /[.:]/.test(segments[0])) { + host = segments[0]; + repository = segments.slice(1).join("/"); + } else if (segments.length === 1) { + repository = `library/${name}`; + } + return { host, repository, tag, ref }; +} + +export function isImmutableTag(ref) { + return IMMUTABLE_TAG_RE.test(parseImageRef(ref).tag); +} + +// Partition a metadata-action tag list into the tags that always push and the +// tags that need an ordering check. +export function classifyTags(refs) { + const immutable = []; + const mutable = []; + for (const ref of refs) { + (isImmutableTag(ref) ? immutable : mutable).push(ref); + } + return { immutable, mutable }; +} + +// Decide a single mutable tag. +// +// `current` is the outcome of resolving the tag's present revision: +// { status: "absent" } tag has never been pushed +// { status: "unlabeled" } image carries no revision label +// { status: "error", detail } registry lookup failed +// { status: "ok", revision } revision is known +// +// `comparison` is the outcome of comparing that revision to the incoming one: +// { status: "ahead" | "identical" } incoming descends from current +// { status: "behind" | "diverged" } incoming is older or unrelated +// { status: "unresolvable" } a commit is no longer in the repo +// { status: "error", detail } compare API failed +// +// Fail-open cases (unlabeled, unresolvable, lookup errors) push with a loud +// warning rather than blocking. A guard that fails closed on a registry or API +// blip would itself stop shipping merged fixes, which is the bug we are fixing, +// only louder. Every fail-open path is annotated so it is visible in the run. +export function decideMutableTag({ tag, incomingSha, current, comparison }) { + const short = (incomingSha || "").slice(0, 7); + switch (current.status) { + case "absent": + return { + push: true, + severity: "info", + reason: `${tag}: tag does not exist yet, publishing ${short}`, + }; + case "unlabeled": + return { + push: true, + severity: "warning", + reason: + `${tag}: current image carries no ${REVISION_LABEL} label, ` + + `cannot verify ordering, publishing ${short} anyway`, + }; + case "error": + return { + push: true, + severity: "warning", + reason: + `${tag}: could not read the current image (${current.detail}), ` + + `cannot verify ordering, publishing ${short} anyway`, + }; + } + + const from = (current.revision || "").slice(0, 7); + switch (comparison.status) { + case "identical": + return { + push: true, + severity: "info", + reason: `${tag}: already at ${short}, republishing the same commit`, + }; + case "ahead": + return { + push: true, + severity: "info", + reason: `${tag}: ${short} descends from ${from}, advancing the tag`, + }; + case "behind": + return { + push: false, + severity: "warning", + reason: + `${tag}: REFUSING to move the tag backwards: ${short} is an ` + + `ancestor of the published ${from}. A newer build already won this ` + + `race; this straggler would un-ship it. Pin sha-${short} if you ` + + `need this exact build.`, + }; + case "diverged": + return { + push: false, + severity: "warning", + reason: + `${tag}: REFUSING to move the tag: ${short} and the published ` + + `${from} have no ancestor relationship. Pin sha-${short} if you ` + + `need this exact build.`, + }; + case "unresolvable": + return { + push: true, + severity: "warning", + reason: + `${tag}: the published revision ${from} is no longer reachable in ` + + `the repository, cannot verify ordering, publishing ${short} anyway`, + }; + default: + return { + push: true, + severity: "warning", + reason: + `${tag}: commit comparison failed (${comparison.detail}), ` + + `cannot verify ordering, publishing ${short} anyway`, + }; + } +} + +// Read the org.opencontainers.image.revision label off whatever a tag +// currently points at. Both repositories are public, so anonymous pull tokens +// are enough and the guard needs no registry credentials of its own. +export async function resolveTagRevision(ref, { fetchImpl = fetch } = {}) { + const { host, repository, tag } = parseImageRef(ref); + const registry = host === "docker.io" ? "registry-1.docker.io" : host; + const base = `https://${registry}/v2/${repository}`; + + let auth = {}; + try { + if (host === "docker.io") { + const tokenRes = await fetchImpl( + `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repository}:pull`, + ); + if (!tokenRes.ok) { + return { status: "error", detail: `docker hub token ${tokenRes.status}` }; + } + const { token } = await tokenRes.json(); + auth = { Authorization: `Bearer ${token}` }; + } + + const manifestRes = await fetchImpl(`${base}/manifests/${tag}`, { + headers: { Accept: MANIFEST_ACCEPT, ...auth }, + }); + if (manifestRes.status === 404) { + return { status: "absent" }; + } + if (!manifestRes.ok) { + return { status: "error", detail: `manifest ${manifestRes.status}` }; + } + let manifest = await manifestRes.json(); + + // Multi-arch: every leg of one build is stamped with the same revision, so + // the first child is representative. + if (Array.isArray(manifest.manifests)) { + const child = manifest.manifests.find((m) => m.digest); + if (!child) { + return { status: "unlabeled" }; + } + const childRes = await fetchImpl(`${base}/manifests/${child.digest}`, { + headers: { Accept: MANIFEST_ACCEPT, ...auth }, + }); + if (!childRes.ok) { + return { status: "error", detail: `child manifest ${childRes.status}` }; + } + manifest = await childRes.json(); + } + + const configDigest = manifest.config && manifest.config.digest; + if (!configDigest) { + return { status: "unlabeled" }; + } + const configRes = await fetchImpl(`${base}/blobs/${configDigest}`, { + headers: auth, + }); + if (!configRes.ok) { + return { status: "error", detail: `config blob ${configRes.status}` }; + } + const config = await configRes.json(); + const labels = + (config.config && config.config.Labels) || + (config.container_config && config.container_config.Labels) || + {}; + const revision = labels[REVISION_LABEL]; + return revision ? { status: "ok", revision } : { status: "unlabeled" }; + } catch (err) { + return { status: "error", detail: String(err && err.message ? err.message : err) }; + } +} + +// Ask GitHub how `head` relates to `base`. The compare API answers this +// directly, which keeps the guard working in the shallow sparse checkouts the +// merge and publish jobs use. `git merge-base --is-ancestor` would need the +// full history fetched into every one of the ~200 merge jobs per push. +export async function compareCommits({ + repository, + base, + head, + token, + fetchImpl = fetch, +}) { + if (base === head) { + return { status: "identical" }; + } + try { + const headers = { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }; + if (token) { + headers.Authorization = `Bearer ${token}`; + } + const res = await fetchImpl( + `https://api.github.com/repos/${repository}/compare/${base}...${head}`, + { headers }, + ); + if (res.status === 404) { + return { status: "unresolvable" }; + } + if (!res.ok) { + return { status: "error", detail: `compare ${res.status}` }; + } + const body = await res.json(); + // GitHub reports ahead/behind/identical/diverged. Unrelated histories in + // the same repository come back as "diverged" with no merge base. + const known = ["ahead", "behind", "identical", "diverged"]; + return known.includes(body.status) + ? { status: body.status } + : { status: "error", detail: `unexpected compare status ${body.status}` }; + } catch (err) { + return { status: "error", detail: String(err && err.message ? err.message : err) }; + } +} + +// Full pass over a metadata-action tag list. Returns the tags that may be +// pushed plus a decision log for the job summary. +export async function guardTags({ + refs, + incomingSha, + repository, + token, + fetchImpl = fetch, +}) { + const { immutable, mutable } = classifyTags(refs); + const decisions = immutable.map((tag) => ({ + tag, + push: true, + severity: "info", + reason: `${tag}: immutable per-commit tag, always published`, + })); + + for (const tag of mutable) { + const current = await resolveTagRevision(tag, { fetchImpl }); + let comparison = { status: "skipped" }; + if (current.status === "ok") { + comparison = await compareCommits({ + repository, + base: current.revision, + head: incomingSha, + token, + fetchImpl, + }); + } + decisions.push({ + tag, + ...decideMutableTag({ tag, incomingSha, current, comparison }), + }); + } + + return { + allowed: decisions.filter((d) => d.push).map((d) => d.tag), + blocked: decisions.filter((d) => !d.push).map((d) => d.tag), + decisions, + }; +} diff --git a/scripts/lib/tag-guard_test.mjs b/scripts/lib/tag-guard_test.mjs new file mode 100644 index 000000000..637162405 --- /dev/null +++ b/scripts/lib/tag-guard_test.mjs @@ -0,0 +1,285 @@ +// Unit tests for the mutable-tag ordering guard (scripts/lib/tag-guard.mjs). +// +// Run with `make test-ci-scripts` (plain `node --test`, no dependencies), which +// is what .github/workflows/lint.yml runs. The registry and GitHub lookups take +// an injected fetch, so nothing here touches the network. + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + classifyTags, + compareCommits, + decideMutableTag, + guardTags, + isImmutableTag, + parseImageRef, +} from "./tag-guard.mjs"; + +const QUAY = "quay.io/go-skynet/local-ai-backends"; +const HUB = "localai/localai-backends"; +const SUFFIX = "-nvidia-l4t-cuda-13-arm64-longcat-video"; + +// The commits from the incident this guard exists to prevent. +const PRE_FIX = "10211948b5470d332a93c841a9c8fe2b9a737148"; +const POST_FIX = "626ae4d51000000000000000000000000000000a"; + +test("parseImageRef splits a registry-qualified ref", () => { + assert.deepEqual(parseImageRef(`${QUAY}:master${SUFFIX}`), { + host: "quay.io", + repository: "go-skynet/local-ai-backends", + tag: `master${SUFFIX}`, + ref: `${QUAY}:master${SUFFIX}`, + }); +}); + +test("parseImageRef treats a dotless first segment as Docker Hub", () => { + const parsed = parseImageRef(`${HUB}:latest${SUFFIX}`); + assert.equal(parsed.host, "docker.io"); + assert.equal(parsed.repository, "localai/localai-backends"); +}); + +test("parseImageRef rejects a ref with no tag", () => { + assert.throws(() => parseImageRef(QUAY), /no tag/); +}); + +test("sha- tags are immutable, rolling tags are not", () => { + assert.equal(isImmutableTag(`${QUAY}:sha-626ae4d${SUFFIX}`), true); + assert.equal(isImmutableTag(`${QUAY}:sha-626ae4d`), true); + assert.equal(isImmutableTag(`${QUAY}:master${SUFFIX}`), false); + assert.equal(isImmutableTag(`${QUAY}:latest${SUFFIX}`), false); + assert.equal(isImmutableTag(`${QUAY}:v4.7.0${SUFFIX}`), false); + // A tag that merely starts with the letters "sha" is not a commit tag. + assert.equal(isImmutableTag(`${QUAY}:shazam${SUFFIX}`), false); +}); + +test("classifyTags partitions a metadata-action tag list", () => { + const { immutable, mutable } = classifyTags([ + `${QUAY}:master${SUFFIX}`, + `${QUAY}:sha-626ae4d${SUFFIX}`, + `${HUB}:master${SUFFIX}`, + `${HUB}:sha-626ae4d${SUFFIX}`, + ]); + assert.deepEqual(immutable, [ + `${QUAY}:sha-626ae4d${SUFFIX}`, + `${HUB}:sha-626ae4d${SUFFIX}`, + ]); + assert.deepEqual(mutable, [`${QUAY}:master${SUFFIX}`, `${HUB}:master${SUFFIX}`]); +}); + +test("a descendant commit advances the tag", () => { + const d = decideMutableTag({ + tag: "master", + incomingSha: POST_FIX, + current: { status: "ok", revision: PRE_FIX }, + comparison: { status: "ahead" }, + }); + assert.equal(d.push, true); + assert.match(d.reason, /advancing the tag/); +}); + +test("the incident case is blocked: an older commit cannot move the tag back", () => { + const d = decideMutableTag({ + tag: `master${SUFFIX}`, + incomingSha: PRE_FIX, + current: { status: "ok", revision: POST_FIX }, + comparison: { status: "behind" }, + }); + assert.equal(d.push, false); + assert.equal(d.severity, "warning"); + assert.match(d.reason, /REFUSING to move the tag backwards/); + // The workaround for the real incident was pinning the sha- tag; the message + // has to point there. + assert.match(d.reason, /Pin sha-1021194/); +}); + +test("unrelated histories are blocked", () => { + const d = decideMutableTag({ + tag: "master", + incomingSha: POST_FIX, + current: { status: "ok", revision: PRE_FIX }, + comparison: { status: "diverged" }, + }); + assert.equal(d.push, false); + assert.match(d.reason, /no ancestor relationship/); +}); + +test("a republish of the same commit is allowed", () => { + const d = decideMutableTag({ + tag: "master", + incomingSha: POST_FIX, + current: { status: "ok", revision: POST_FIX }, + comparison: { status: "identical" }, + }); + assert.equal(d.push, true); + assert.equal(d.severity, "info"); +}); + +test("a tag that does not exist yet is published without a warning", () => { + const d = decideMutableTag({ + tag: "v4.7.0", + incomingSha: POST_FIX, + current: { status: "absent" }, + comparison: { status: "skipped" }, + }); + assert.equal(d.push, true); + assert.equal(d.severity, "info"); +}); + +test("fail-open paths push but always warn", () => { + for (const [current, comparison, pattern] of [ + [{ status: "unlabeled" }, { status: "skipped" }, /no org\.opencontainers\.image\.revision label/], + [{ status: "error", detail: "manifest 503" }, { status: "skipped" }, /manifest 503/], + [{ status: "ok", revision: PRE_FIX }, { status: "unresolvable" }, /no longer reachable/], + [{ status: "ok", revision: PRE_FIX }, { status: "error", detail: "compare 502" }, /compare 502/], + ]) { + const d = decideMutableTag({ + tag: "master", + incomingSha: POST_FIX, + current, + comparison, + }); + assert.equal(d.push, true); + assert.equal(d.severity, "warning"); + assert.match(d.reason, pattern); + } +}); + +// --- fetch-backed helpers ------------------------------------------------- + +function jsonResponse(body, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + }; +} + +test("compareCommits short-circuits identical commits without a request", async () => { + let called = false; + const res = await compareCommits({ + repository: "mudler/LocalAI", + base: POST_FIX, + head: POST_FIX, + fetchImpl: async () => { + called = true; + return jsonResponse({}); + }, + }); + assert.deepEqual(res, { status: "identical" }); + assert.equal(called, false); +}); + +test("compareCommits maps a 404 to unresolvable", async () => { + const res = await compareCommits({ + repository: "mudler/LocalAI", + base: PRE_FIX, + head: POST_FIX, + fetchImpl: async () => jsonResponse({}, 404), + }); + assert.deepEqual(res, { status: "unresolvable" }); +}); + +test("compareCommits surfaces the GitHub status verbatim", async () => { + const res = await compareCommits({ + repository: "mudler/LocalAI", + base: POST_FIX, + head: PRE_FIX, + fetchImpl: async () => jsonResponse({ status: "behind" }), + }); + assert.deepEqual(res, { status: "behind" }); +}); + +test("compareCommits turns a thrown fetch into a non-fatal error", async () => { + const res = await compareCommits({ + repository: "mudler/LocalAI", + base: PRE_FIX, + head: POST_FIX, + fetchImpl: async () => { + throw new Error("ECONNRESET"); + }, + }); + assert.equal(res.status, "error"); + assert.match(res.detail, /ECONNRESET/); +}); + +// A fake registry serving one multi-arch tag stamped with `revision`. +function fakeRegistry(revision) { + return async (url) => { + if (url.includes("auth.docker.io")) { + return jsonResponse({ token: "t" }); + } + if (url.endsWith("/manifests/master-x")) { + return jsonResponse({ manifests: [{ digest: "sha256:child" }] }); + } + if (url.endsWith("/manifests/sha256:child")) { + return jsonResponse({ config: { digest: "sha256:cfg" } }); + } + if (url.endsWith("/blobs/sha256:cfg")) { + return jsonResponse({ + config: { Labels: { "org.opencontainers.image.revision": revision } }, + }); + } + return jsonResponse({}, 404); + }; +} + +test("guardTags always keeps immutable tags and drops a backwards mutable tag", async () => { + const fetchImpl = async (url, init) => { + if (url.startsWith("https://api.github.com/")) { + return jsonResponse({ status: "behind" }); + } + return fakeRegistry(POST_FIX)(url, init); + }; + + const result = await guardTags({ + refs: [`${QUAY}:sha-1021194-x`, `${QUAY}:master-x`], + incomingSha: PRE_FIX, + repository: "mudler/LocalAI", + fetchImpl, + }); + + assert.deepEqual(result.allowed, [`${QUAY}:sha-1021194-x`]); + assert.deepEqual(result.blocked, [`${QUAY}:master-x`]); +}); + +test("guardTags advances a mutable tag when the incoming commit is newer", async () => { + const fetchImpl = async (url, init) => { + if (url.startsWith("https://api.github.com/")) { + return jsonResponse({ status: "ahead" }); + } + return fakeRegistry(PRE_FIX)(url, init); + }; + + const result = await guardTags({ + refs: [`${QUAY}:sha-626ae4d-x`, `${QUAY}:master-x`], + incomingSha: POST_FIX, + repository: "mudler/LocalAI", + fetchImpl, + }); + + assert.deepEqual(result.allowed, [`${QUAY}:sha-626ae4d-x`, `${QUAY}:master-x`]); + assert.deepEqual(result.blocked, []); +}); + +test("guardTags publishes an unlabeled tag rather than stalling on it", async () => { + const fetchImpl = async (url) => { + if (url.endsWith("/manifests/master-x")) { + return jsonResponse({ config: { digest: "sha256:cfg" } }); + } + if (url.endsWith("/blobs/sha256:cfg")) { + return jsonResponse({ config: {} }); + } + return jsonResponse({}, 404); + }; + + const result = await guardTags({ + refs: [`${QUAY}:master-x`], + incomingSha: POST_FIX, + repository: "mudler/LocalAI", + fetchImpl, + }); + + assert.deepEqual(result.allowed, [`${QUAY}:master-x`]); + assert.equal(result.decisions[0].severity, "warning"); +}); diff --git a/scripts/tag-guard.mjs b/scripts/tag-guard.mjs new file mode 100644 index 000000000..b5f760819 --- /dev/null +++ b/scripts/tag-guard.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +// +// CI entrypoint for the mutable-tag ordering guard. Reads the tag list +// docker/metadata-action produced, decides which tags this build is allowed to +// publish (see scripts/lib/tag-guard.mjs for the rules and the incident that +// motivated them), prints the surviving tags one per line on stdout, and logs +// every decision to stderr, the GitHub job summary and, for anything blocked or +// unverifiable, a workflow annotation. +// +// The whole point of this guard is to be noisy: the bug it fixes ran silently +// for two days. Never make a skipped tag a quiet no-op. +// +// Inputs (environment): +// DOCKER_METADATA_OUTPUT_JSON metadata-action output; .tags is used +// TAGS newline/comma separated tags (alternative) +// GITHUB_SHA commit being published +// GITHUB_REPOSITORY owner/name, for the compare API +// GITHUB_TOKEN optional, raises the compare API rate limit +// REGISTRY_PREFIX optional, keep only tags with this prefix +// +// Exit codes: +// 0 decisions made (blocked tags are not an error; the build still ships +// its immutable sha- tag, which is the artifact humans pin) +// 1 bad input, e.g. no tag list at all + +import fs from "node:fs"; + +import { guardTags } from "./lib/tag-guard.mjs"; + +function readTags() { + const raw = process.env.DOCKER_METADATA_OUTPUT_JSON; + if (raw) { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed.tags)) { + return parsed.tags; + } + } + if (process.env.TAGS) { + return process.env.TAGS.split(/[\n,]/) + .map((t) => t.trim()) + .filter(Boolean); + } + return []; +} + +function appendSummary(lines) { + const path = process.env.GITHUB_STEP_SUMMARY; + if (!path) { + return; + } + try { + fs.appendFileSync(path, `${lines.join("\n")}\n`); + } catch (err) { + process.stderr.write(`tag-guard: could not write job summary: ${err}\n`); + } +} + +async function main() { + const prefix = process.env.REGISTRY_PREFIX || ""; + const refs = readTags().filter((t) => t.startsWith(prefix)); + if (refs.length === 0) { + process.stderr.write( + `tag-guard: no tags to consider (REGISTRY_PREFIX=${prefix || ""})\n`, + ); + return; + } + + const incomingSha = process.env.GITHUB_SHA; + if (!incomingSha) { + throw new Error("GITHUB_SHA is not set"); + } + + const { allowed, blocked, decisions } = await guardTags({ + refs, + incomingSha, + repository: process.env.GITHUB_REPOSITORY || "mudler/LocalAI", + token: process.env.GITHUB_TOKEN, + }); + + const summary = ["", `#### Tag ordering guard (${incomingSha.slice(0, 7)})`, ""]; + for (const d of decisions) { + process.stderr.write(`tag-guard: ${d.push ? "PUSH " : "SKIP "} ${d.reason}\n`); + summary.push(`- ${d.push ? "published" : "**skipped**"}: ${d.reason}`); + // Annotate anything that is not a plain, verified advance so it shows up on + // the run page without anyone having to open the log. + if (!d.push || d.severity === "warning") { + process.stderr.write(`::warning title=backend tag guard::${d.reason}\n`); + } + } + appendSummary(summary); + + if (blocked.length > 0) { + process.stderr.write( + `tag-guard: ${blocked.length} mutable tag(s) withheld; ` + + `${allowed.length} tag(s) will be published\n`, + ); + } + + process.stdout.write(`${allowed.join("\n")}\n`); +} + +main().catch((err) => { + process.stderr.write(`tag-guard: ${err && err.stack ? err.stack : err}\n`); + process.exit(1); +});