From 3655b042583293ec55675e4f8e9d6e55fafe0b96 Mon Sep 17 00:00:00 2001 From: Artur Neumann Date: Tue, 18 Mar 2025 11:56:56 +0545 Subject: [PATCH] fix finding existing caches in CI --- .woodpecker.star | 45 ++++++++++--------- tests/config/woodpecker/check_go_bin_cache.sh | 12 +++-- tests/config/woodpecker/check_web_cache.sh | 13 +++--- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/.woodpecker.star b/.woodpecker.star index a4b90d1567..3065ba9f4f 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -53,7 +53,7 @@ dirs = { "web": "/woodpecker/src/github.com/opencloud-eu/opencloud/webTestRunner", "zip": "/woodpecker/src/github.com/opencloud-eu/opencloud/zip", "webZip": "/woodpecker/src/github.com/opencloud-eu/opencloud/zip/web.tar.gz", - "webPnpmZip": "/woodpecker/src/github.com/opencloud-eu/opencloud/zip/pnpm-store.tar.gz", + "webPnpmZip": "/woodpecker/src/github.com/opencloud-eu/opencloud/zip/web-pnpm.tar.gz", "baseGo": "/go/src/github.com/opencloud-eu/opencloud", "gobinTar": "go-bin.tar.gz", "gobinTarPath": "/go/src/github.com/opencloud-eu/opencloud/go-bin.tar.gz", @@ -562,15 +562,8 @@ def getGoBinForTesting(ctx): def checkGoBinCache(): return [{ "name": "check-go-bin-cache", - "image": OC_UBUNTU, - "environment": { - "CACHE_ENDPOINT": { - "from_secret": "cache_s3_server", - }, - "CACHE_BUCKET": { - "from_secret": "cache_s3_bucket", - }, - }, + "image": MINIO_MC, + "environment": MINIO_MC_ENV, "commands": [ "bash -x %s/tests/config/woodpecker/check_go_bin_cache.sh %s %s" % (dirs["baseGo"], dirs["baseGo"], dirs["gobinTar"]), ], @@ -582,6 +575,8 @@ def cacheGoBin(): "name": "bingo-get", "image": OC_CI_GOLANG, "commands": [ + ". ./.env", + "if $BIN_CACHE_FOUND; then exit 0; fi", "make bingo-update", ], "environment": CI_HTTP_PROXY_ENV, @@ -590,6 +585,8 @@ def cacheGoBin(): "name": "archive-go-bin", "image": OC_UBUNTU, "commands": [ + ". ./.env", + "if $BIN_CACHE_FOUND; then exit 0; fi", "tar -czvf %s /go/bin" % dirs["gobinTarPath"], ], }, @@ -598,6 +595,8 @@ def cacheGoBin(): "image": MINIO_MC, "environment": MINIO_MC_ENV, "commands": [ + ". ./.env", + "if $BIN_CACHE_FOUND; then exit 0; fi", # .bingo folder will change after 'bingo-get' # so get the stored hash of a .bingo folder "BINGO_HASH=$(cat %s/.bingo_hash)" % dirs["baseGo"], @@ -2640,15 +2639,8 @@ def getWoodpeckerEnvAndCheckScript(ctx): def checkForWebCache(name): return { "name": "check-for-%s-cache" % name, - "image": OC_UBUNTU, - "environment": { - "CACHE_ENDPOINT": { - "from_secret": "cache_s3_server", - }, - "CACHE_BUCKET": { - "from_secret": "cache_s3_bucket", - }, - }, + "image": MINIO_MC, + "environment": MINIO_MC_ENV, "commands": [ "bash -x check_web_cache.sh %s" % name, ], @@ -2660,6 +2652,7 @@ def cloneWeb(): "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, "commands": [ ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", "rm -rf %s" % dirs["web"], "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/opencloud-eu/web.git %s" % dirs["web"], "cd %s && git checkout $WEB_COMMITID" % dirs["web"], @@ -2675,6 +2668,8 @@ def generateWebPnpmCache(ctx): "name": "install-pnpm", "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, "commands": [ + ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", "cd %s" % dirs["web"], 'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"', "pnpm config set store-dir ./.pnpm-store", @@ -2685,6 +2680,8 @@ def generateWebPnpmCache(ctx): "name": "zip-pnpm", "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, "commands": [ + ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", # zip the pnpm deps before caching "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), "cd %s" % dirs["web"], @@ -2696,7 +2693,8 @@ def generateWebPnpmCache(ctx): "image": MINIO_MC, "environment": MINIO_MC_ENV, "commands": [ - "source ./.woodpecker.env", + ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", # cache using the minio/mc client to the public bucket (long term bucket) "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", "mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webPnpmZip"], @@ -2713,6 +2711,8 @@ def generateWebCache(ctx): "name": "zip-web", "image": OC_UBUNTU, "commands": [ + ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), "tar -czvf %s webTestRunner" % dirs["webZip"], ], @@ -2722,7 +2722,8 @@ def generateWebCache(ctx): "image": MINIO_MC, "environment": MINIO_MC_ENV, "commands": [ - "source ./.woodpecker.env", + ". ./.woodpecker.env", + "if $WEB_CACHE_FOUND; then exit 0; fi", # cache using the minio/mc client to the 'owncloud' bucket (long term bucket) "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", "mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webZip"], @@ -2758,7 +2759,7 @@ def restoreWebPnpmCache(): "commands": [ "source ./.woodpecker.env", "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/pnpm-store.tar.gz %s" % dirs["zip"], + "mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/web-pnpm.tar.gz %s" % dirs["zip"], ], }, { # we need to install again because the node_modules are not cached diff --git a/tests/config/woodpecker/check_go_bin_cache.sh b/tests/config/woodpecker/check_go_bin_cache.sh index a665adfd5a..7e1946225b 100644 --- a/tests/config/woodpecker/check_go_bin_cache.sh +++ b/tests/config/woodpecker/check_go_bin_cache.sh @@ -15,13 +15,17 @@ BINGO_DIR="$ROOT_PATH/.bingo" BINGO_HASH=$(cat "$BINGO_DIR"/* | sha256sum | cut -d ' ' -f 1) URL="$CACHE_ENDPOINT/$CACHE_BUCKET/opencloud/go-bin/$BINGO_HASH/$2" -if curl --output /dev/null --silent --head --fail "$URL"; then + +mc alias set s3 "$MC_HOST" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" + +if mc ls --json s3/"$CACHE_BUCKET"/opencloud/go-bin/"$BINGO_HASH"/$2 | grep "\"status\":\"success\""; then echo "[INFO] Go bin cache with has '$BINGO_HASH' exists." - # https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951 - # exit a Pipeline early without failing - exit 78 + ENV="BIN_CACHE_FOUND=true\n" else # stored hash of a .bingo folder to '.bingo_hash' file echo "$BINGO_HASH" >"$ROOT_PATH/.bingo_hash" echo "[INFO] Go bin cache with has '$BINGO_HASH' does not exist." + ENV="BIN_CACHE_FOUND=false\n" fi + +echo -e $ENV >> .env diff --git a/tests/config/woodpecker/check_web_cache.sh b/tests/config/woodpecker/check_web_cache.sh index a57617b6d5..88f4992675 100644 --- a/tests/config/woodpecker/check_web_cache.sh +++ b/tests/config/woodpecker/check_web_cache.sh @@ -10,16 +10,15 @@ fi echo "Checking web version - $WEB_COMMITID in cache" -URL="$CACHE_ENDPOINT/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/$1.tar.gz" +mc alias set s3 "$MC_HOST" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" -echo "Checking for the web cache at '$URL'." - -if curl --output /dev/null --silent --head --fail "$URL" +if mc ls --json s3/"$CACHE_BUCKET"/opencloud/web-test-runner/"$WEB_COMMITID"/"$1".tar.gz | grep "\"status\":\"success\""; then echo "$1 cache with commit id $WEB_COMMITID already available." - # https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951 - # exit a Pipeline early without failing - exit 78 + ENV="WEB_CACHE_FOUND=true\n" else echo "$1 cache with commit id $WEB_COMMITID was not available." + ENV="WEB_CACHE_FOUND=false\n" fi + +echo -e $ENV >> .woodpecker.env