mirror/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-06-28 18:05:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

build: cache Go build and module dirs to speed up CI

Browse Source 
The build matrix relied on setup-go's built-in cache, which keys only on
go.sum with no job differentiation. All matrix jobs computed the same
cache key and raced to save it; since cache keys are immutable, only the
first job to finish saved its cache. That winner was usually a fast job
whose build cache contained none of the cross-compiled architectures, so
the compile_all and ci_beta steps started from a cold cache on every run.

Disable setup-go's cache and add two explicit actions/cache steps to the
build matrix, the android job and the lint job:

  - the module cache (~/go/pkg/mod) depends only on go.sum, so it is
    shared across all jobs under a single key; it used to be duplicated in
    every job's cache. The downloaded module .zip archives are pruned
    before saving as they are not needed to build from the extracted
    module cache, roughly halving it to ~260 MiB per OS;
  - the build cache (compiled artifacts) is specific to OS, arch and Go
    version, so it is kept per job, keyed on the job name.

This lets the cross-compile steps reuse per-architecture build artifacts
and keeps the total cache within the repository limit.

Measured on CI, comparing a cold-cache run against the following
warm-cache run:

    other_os    23m12s -> 3m35s    (compile_all 14m -> 21s)
    linux       23m13s -> 12m14s   (deploy 11m -> 1m37s,
                                     race test 8m -> 4m45s)

Both jobs now finish well under 15 minutes once the cache is warm.
This commit is contained in:
Nick Craig-Wood
2026-05-31 10:30:24 +01:00
parent 5f5e2ef9cf
commit dff3a0731e
1 changed files with 112 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
.github/workflows/build.yml vendored
View File

@@ -97,16 +97,58 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
id: setup-go
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go }}
check-latest: true
# Caching is handled explicitly below. setup-go's built-in cache
# keys only on go.sum, so all matrix jobs collide on one key and
# only the first to finish saves - which loses the cross-compile
# build cache.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
# The module cache holds downloaded source and is identical across
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
# it is shared via a single key to avoid storing the same ~600 MiB in
# every job's cache and overflowing the repo cache limit.
- name: Module cache
uses: actions/cache@v5
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
# The build cache holds compiled artifacts which are specific to the
# OS, arch and Go version, so it is kept per job.
- name: Build cache
uses: actions/cache@v5
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set environment variables
run: |
@@ -205,6 +247,15 @@ jobs:
# Deploy binaries if enabled in config && not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
lint:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -231,11 +282,19 @@ jobs:
check-latest: true
cache: false
- name: Module cache
uses: actions/cache@v5
with:
path: ~/go/pkg/mod
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Cache
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
@@ -302,6 +361,15 @@ jobs:
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
if: github.event_name == 'pull_request'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
android:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -309,6 +377,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v6
with:
@@ -316,9 +390,37 @@ jobs:
# Upgrade together with NDK version
- name: Set up Go
id: setup-go
uses: actions/setup-go@v6
with:
go-version: '~1.26.3'
# Caching is handled explicitly below to share the module cache
# with the other jobs - see the build job for the rationale.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
- name: Module cache
uses: actions/cache@v5
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Build cache
uses: actions/cache@v5
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set global environment variables
run: |
@@ -394,3 +496,12 @@ jobs:
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
# Upload artifacts if not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true