From dff3a0731ea03e51031cebc0d2269f1352dd66dc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 31 May 2026 10:30:24 +0100 Subject: [PATCH] build: cache Go build and module dirs to speed up CI The build matrix relied on setup-go's built-in cache, which keys only on go.sum with no job differentiation. All matrix jobs computed the same cache key and raced to save it; since cache keys are immutable, only the first job to finish saved its cache. That winner was usually a fast job whose build cache contained none of the cross-compiled architectures, so the compile_all and ci_beta steps started from a cold cache on every run. Disable setup-go's cache and add two explicit actions/cache steps to the build matrix, the android job and the lint job: - the module cache (~/go/pkg/mod) depends only on go.sum, so it is shared across all jobs under a single key; it used to be duplicated in every job's cache. The downloaded module .zip archives are pruned before saving as they are not needed to build from the extracted module cache, roughly halving it to ~260 MiB per OS; - the build cache (compiled artifacts) is specific to OS, arch and Go version, so it is kept per job, keyed on the job name. This lets the cross-compile steps reuse per-architecture build artifacts and keeps the total cache within the repository limit. Measured on CI, comparing a cold-cache run against the following warm-cache run: other_os 23m12s -> 3m35s (compile_all 14m -> 21s) linux 23m13s -> 12m14s (deploy 11m -> 1m37s, race test 8m -> 4m45s) Both jobs now finish well under 15 minutes once the cache is warm. --- .github/workflows/build.yml | 113 +++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1792fa32..9dacf0ebf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,16 +97,58 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Get runner parameters + id: get-runner-parameters + run: | + echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT + echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT + - name: Checkout uses: actions/checkout@v6 with: fetch-depth: 0 - name: Install Go + id: setup-go uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} check-latest: true + # Caching is handled explicitly below. setup-go's built-in cache + # keys only on go.sum, so all matrix jobs collide on one key and + # only the first to finish saves - which loses the cross-compile + # build cache. + cache: false + + - name: Get Go paths + id: go-paths + run: | + echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT + + # The module cache holds downloaded source and is identical across + # jobs (it depends only on go.sum, not on OS, arch or Go version), so + # it is shared via a single key to avoid storing the same ~600 MiB in + # every job's cache and overflowing the repo cache limit. + - name: Module cache + uses: actions/cache@v5 + with: + path: ${{ steps.go-paths.outputs.mod-cache }} + key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-mod-${{ steps.get-runner-parameters.outputs.year-week }}- + go-mod- + + # The build cache holds compiled artifacts which are specific to the + # OS, arch and Go version, so it is kept per job. + - name: Build cache + uses: actions/cache@v5 + with: + path: ${{ steps.go-paths.outputs.build-cache }} + key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}- + go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}- - name: Set environment variables run: | @@ -205,6 +247,15 @@ jobs: # Deploy binaries if enabled in config && not a PR && not a fork if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' + - name: Trim module cache before saving + if: always() + run: | + # The downloaded module .zip archives are not needed to build from + # the already-extracted module cache, so drop them before the cache + # is saved - this roughly halves the module cache size. + modcache=$(go env GOMODCACHE | tr '\\' '/') + find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true + lint: if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) timeout-minutes: 30 @@ -231,11 +282,19 @@ jobs: check-latest: true cache: false + - name: Module cache + uses: actions/cache@v5 + with: + path: ~/go/pkg/mod + key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-mod-${{ steps.get-runner-parameters.outputs.year-week }}- + go-mod- + - name: Cache uses: actions/cache@v5 with: path: | - ~/go/pkg/mod ~/.cache/go-build ~/.cache/golangci-lint key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} @@ -302,6 +361,15 @@ jobs: run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}' if: github.event_name == 'pull_request' + - name: Trim module cache before saving + if: always() + run: | + # The downloaded module .zip archives are not needed to build from + # the already-extracted module cache, so drop them before the cache + # is saved - this roughly halves the module cache size. + modcache=$(go env GOMODCACHE | tr '\\' '/') + find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true + android: if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) timeout-minutes: 30 @@ -309,6 +377,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Get runner parameters + id: get-runner-parameters + run: | + echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT + echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT + - name: Checkout uses: actions/checkout@v6 with: @@ -316,9 +390,37 @@ jobs: # Upgrade together with NDK version - name: Set up Go + id: setup-go uses: actions/setup-go@v6 with: go-version: '~1.26.3' + # Caching is handled explicitly below to share the module cache + # with the other jobs - see the build job for the rationale. + cache: false + + - name: Get Go paths + id: go-paths + run: | + echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT + + - name: Module cache + uses: actions/cache@v5 + with: + path: ${{ steps.go-paths.outputs.mod-cache }} + key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-mod-${{ steps.get-runner-parameters.outputs.year-week }}- + go-mod- + + - name: Build cache + uses: actions/cache@v5 + with: + path: ${{ steps.go-paths.outputs.build-cache }} + key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}- + go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}- - name: Set global environment variables run: | @@ -394,3 +496,12 @@ jobs: RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} # Upload artifacts if not a PR && not a fork if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone' + + - name: Trim module cache before saving + if: always() + run: | + # The downloaded module .zip archives are not needed to build from + # the already-extracted module cache, so drop them before the cache + # is saved - this roughly halves the module cache size. + modcache=$(go env GOMODCACHE | tr '\\' '/') + find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true