diff --git a/.drone.star b/.drone.star index 610f6b59b..faf7182e7 100644 --- a/.drone.star +++ b/.drone.star @@ -234,6 +234,7 @@ def main(ctx): cancelPreviousBuilds() + \ codestyle(ctx) + \ buildWebCache(ctx) + \ + getGoBinForTesting(ctx) + \ [buildOcisBinaryForTesting(ctx)] + \ testOcisModules(ctx) + \ testPipelines(ctx) @@ -364,8 +365,89 @@ def testPipelines(ctx): return pipelines +def getGoBinForTesting(ctx): + return [{ + "kind": "pipeline", + "type": "docker", + "name": "get-go-bin-cache", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "steps": skipIfUnchanged(ctx, "unit-tests") + + checkGoBinCache() + + cacheGoBin(), + "trigger": { + "ref": [ + "refs/heads/master", + "refs/heads/stable-*", + "refs/pull/**", + ], + }, + "volumes": [pipelineVolumeGo], + }] + +def checkGoBinCache(): + return [{ + "name": "check-go-bin-cache", + "image": OC_UBUNTU, + "environment": { + "CACHE_ENDPOINT": { + "from_secret": "cache_public_s3_server", + }, + "CACHE_BUCKET": { + "from_secret": "cache_public_s3_bucket", + }, + }, + "commands": [ + "bash -x %s/tests/config/drone/check_go_bin_cache.sh %s" % (dirs["base"], dirs["base"]), + ], + }] + +def cacheGoBin(): + return [ + { + "name": "bingo-get", + "image": OC_CI_GOLANG, + "commands": [ + "make bingo-update", + ], + "volumes": [stepVolumeGo], + }, + { + "name": "cache-go-bin", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + # .bingo folder will change after 'bingo-get' + # so get the stored hash of a .bingo folder + "BINGO_HASH=$(cat %s/.bingo_hash)" % dirs["base"], + # cache using the minio client to the public bucket (long term bucket) + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc cp -r /go/bin s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH", + ], + "volumes": [stepVolumeGo], + }, + ] + +def restoreGoBinCache(): + return [ + { + "name": "restore-go-bin-cache", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "BINGO_HASH=$(cat %s/.bingo/* | sha256sum | cut -d ' ' -f 1)" % dirs["base"], + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc cp -r -a s3/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH/bin /go", + "chmod +x /go/bin/*", + ], + "volumes": [stepVolumeGo], + }, + ] + def testOcisModule(ctx, module): - steps = skipIfUnchanged(ctx, "unit-tests") + makeGoGenerate(module) + [ + steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate(module) + [ { "name": "golangci-lint", "image": OC_CI_GOLANG, @@ -423,6 +505,7 @@ def testOcisModule(ctx, module): "refs/pull/**", ], }, + "depends_on": getPipelineNames(getGoBinForTesting(ctx)), "volumes": [pipelineVolumeGo], } diff --git a/Makefile b/Makefile index 7e5a0a220..1e30910b7 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ golangci-lint-fix: .PHONY: bingo-update bingo-update: $(BINGO) - $(BINGO) get -l -u + $(BINGO) get -l -v .PHONY: check-licenses check-licenses: ci-go-check-licenses ci-node-check-licenses diff --git a/tests/config/drone/check_go_bin_cache.sh b/tests/config/drone/check_go_bin_cache.sh new file mode 100644 index 000000000..6d542d6ff --- /dev/null +++ b/tests/config/drone/check_go_bin_cache.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# +# $1 - root path where .bingo resides +# + +ROOT_PATH="$1" +if [ -z "$1" ]; then + ROOT_PATH="/drone/src" +fi +BINGO_DIR="$ROOT_PATH/.bingo" + +# generate hash of a .bingo folder +BINGO_HASH=$(cat "$BINGO_DIR"/* | sha256sum | cut -d ' ' -f 1) + +URL="$CACHE_ENDPOINT/$CACHE_BUCKET/ocis/go-bin/$BINGO_HASH/bin/golangci-lint" +if curl --output /dev/null --silent --head --fail "$URL"; then + echo "[INFO] Go bin cache with has '$BINGO_HASH' exists." + # https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951 + # exit a Pipeline early without failing + exit 78 +else + # stored hash of a .bingo folder to '.bingo_hash' file + echo "$BINGO_HASH" >"$ROOT_PATH/.bingo_hash" + echo "[INFO] Go bin cache with has '$BINGO_HASH' does not exist." +fi