diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6cdd677cc..4dbe1383c 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -359,6 +359,21 @@ GitHub Actions caches are limited to 10 GB per repo. Steady-state worst case: ~8 One residual self-hosted reference remains in `test-extra.yml` (`tests-vibevoice-cpp-grpc-transcription` uses `bigger-runner` for the 30s JFK-decode timeout headroom). That's a separate concern. +### Small always-on jobs routed to `arc-runner-set` + +The hosted pool is shared across the whole *account*, not per repo, so a burst in one repo starves the others. On 2026-07-31 it went to **zero scheduled jobs for 35 consecutive minutes** with 39 jobs queued, while `arc-runner-set` completed 12 jobs without interruption over the same window. Actions was healthy globally at the time (other public repos were scheduling normally), so this is an account-level throttle, not an outage. + +Two small, high-frequency workflows are therefore routed to `arc-runner-set`: + +| workflow | jobs | routing expression selects self-hosted when | +|---|---|---| +| `gh-pages.yml` | `build`, `deploy` | `github.repository == 'mudler/LocalAI'` | +| `lint.yml` | `golangci-lint`, `build-scripts` | `github.event_name == 'push'` **and** the repo guard | + +The `lint.yml` routing is push-only **on purpose**. That workflow also triggers on `pull_request`, and a fork PR runs untrusted contributor code; that must stay on the ephemeral hosted pool and never touch a self-hosted runner. `gh-pages.yml` needs no such clause because it only triggers on push-to-master and `workflow_dispatch`. The repository guard in both keeps forks (which have no such runner label) from queueing forever. + +Both workflows fetch their own toolchains (`setup-go`, `actions-hugo`) and use no `sudo`/`apt`. Because a self-hosted image may be leaner than the hosted one, each `lint.yml` job opens with a preflight step that names any missing tool (`curl`/`unzip`/`make` for protoc and lint, `gcc`/`ldd`/`python3` for the packaging-script tests) instead of failing opaquely mid-build. If the runner image turns out to lack them, either extend the image or revert the single `runs-on:` expression per job. + ## Touching the cache pipeline When changing `image_build.yml`, `backend_build.yml`, any of the `backend/Dockerfile.*` files, `Dockerfile.base-grpc-builder`, `.docker/install-base-deps.sh`, `.docker/-compile.sh`, or `scripts/changed-backends.js`: diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 9db309c53..abcd0aa45 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -25,7 +25,20 @@ concurrency: jobs: build: - runs-on: ubuntu-latest + # Self-hosted. This workflow is push-to-master + workflow_dispatch only, so + # it never executes pull-request code and a fork cannot reach the runner + # with untrusted changes. The repository guard keeps forks (whose own master + # pushes would otherwise queue forever against a label they do not have) on + # the hosted pool. + # + # Why: the GitHub-hosted pool is shared account-wide and has repeatedly + # starved (2026-07-31: 35 consecutive minutes at zero scheduled jobs, while + # arc-runner-set kept completing work throughout). Publishing the site is + # small, frequent, and must not sit behind a saturated hosted queue. + # + # Needs only git, tar and curl on the runner: setup-go and actions-hugo + # fetch their own toolchains, and no step uses sudo, apt, make or unzip. + runs-on: ${{ github.repository == 'mudler/LocalAI' && 'arc-runner-set' || 'ubuntu-latest' }} env: HUGO_VERSION: "0.146.3" steps: @@ -86,7 +99,11 @@ jobs: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + # Same routing as build: a hosted slot for a ~10s deploy is exactly the kind + # of job that should not block on a starved pool. deploy-pages authenticates + # with the job's OIDC token (id-token: write above), which self-hosted + # runners issue the same way hosted ones do. + runs-on: ${{ github.repository == 'mudler/LocalAI' && 'arc-runner-set' || 'ubuntu-latest' }} needs: build steps: - name: Deploy to GitHub Pages diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1e32031b0..04d31985e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,8 +21,36 @@ concurrency: jobs: golangci-lint: - runs-on: ubuntu-latest + # Self-hosted for PUSH only, and only in the canonical repo. + # + # This workflow also runs on pull_request, which for a fork PR means + # executing untrusted contributor code. That must never land on a + # self-hosted runner, so anything that is not a push to mudler/LocalAI stays + # on the ephemeral hosted pool. Pushes to master are trusted code that has + # already been reviewed and merged. + # + # Why at all: the hosted pool is shared account-wide and starved for 35 + # straight minutes on 2026-07-31 while arc-runner-set kept completing jobs. + # Lint is small and runs on every commit, so it is a good candidate to move + # off the contended pool. + runs-on: ${{ (github.event_name == 'push' && github.repository == 'mudler/LocalAI') && 'arc-runner-set' || 'ubuntu-latest' }} steps: + - name: Preflight - required host tools + # The hosted images ship these; a self-hosted container image may not. + # Check up front so a missing tool reports itself by name instead of + # surfacing as an opaque failure inside `make protogen-go` (which needs + # curl + unzip for protoc) or `make lint`. + run: | + missing="" + for t in git curl unzip make tar; do + command -v "$t" >/dev/null 2>&1 || missing="$missing $t" + done + echo "runner: ${RUNNER_NAME:-unknown} os: $(uname -sm)" + if [ -n "$missing" ]; then + echo "::error::missing required tools on this runner:$missing" + exit 1 + fi + echo "all required tools present" - uses: actions/checkout@v7 with: # Full history so golangci-lint's new-from-merge-base can reach @@ -55,8 +83,25 @@ jobs: # container build (a missing transitive dep, a partial cuDNN family). Their # shell tests need nothing but bash + gcc + ldd, so run them on every PR # rather than waiting on a multi-GB cross-arch backend image build. - runs-on: ubuntu-latest + # + # Push-only self-hosted routing, same fork-safety reasoning as + # golangci-lint above. + runs-on: ${{ (github.event_name == 'push' && github.repository == 'mudler/LocalAI') && 'arc-runner-set' || 'ubuntu-latest' }} steps: + - name: Preflight - required host tools + # This job additionally needs a C toolchain: the packaging-script tests + # compile a throwaway binary and inspect it with ldd. + run: | + missing="" + for t in git make gcc ldd python3; do + command -v "$t" >/dev/null 2>&1 || missing="$missing $t" + done + echo "runner: ${RUNNER_NAME:-unknown} os: $(uname -sm)" + if [ -n "$missing" ]; then + echo "::error::missing required tools on this runner:$missing" + exit 1 + fi + echo "all required tools present" - uses: actions/checkout@v7 - name: run packaging script tests run: make test-build-scripts