diff --git a/.drone.star b/.drone.star index 04b0fefcd0..14703a2567 100644 --- a/.drone.star +++ b/.drone.star @@ -276,15 +276,15 @@ def main(ctx): pipelineSanityChecks(ctx, pipelines) return pipelines -def buildWebCache(ctx): - return [{ +def cachePipeline(name, steps): + return { "kind": "pipeline", "type": "docker", - "name": "build-web-cache", + "name": "build-%s-cache" % name, "clone": { "disable": True, }, - "steps": generateWebCache(ctx), + "steps": steps, "trigger": { "ref": [ "refs/heads/master", @@ -292,7 +292,14 @@ def buildWebCache(ctx): "refs/pull/**", ], }, - }] + } + +def buildWebCache(ctx): + return [ + cachePipeline("web", generateWebCache(ctx)), + cachePipeline("web-acceptance", generateWebAcceptanceCache(ctx)), + cachePipeline("web-e2e", generateWebE2ECache(ctx)), + ] def testOcisModules(ctx): pipelines = [] @@ -2870,45 +2877,127 @@ def setupForLitmus(): ], }] -def generateWebCache(ctx): +def getDroneEnvAndCheckScript(ctx): ocis_git_base_url = "https://raw.githubusercontent.com/owncloud/ocis" path_to_drone_env = "%s/%s/.drone.env" % (ocis_git_base_url, ctx.build.commit) path_to_check_script = "%s/%s/tests/config/drone/check_web_cache.sh" % (ocis_git_base_url, ctx.build.commit) + return { + "name": "get-drone-env-and-check-script", + "image": OC_UBUNTU, + "commands": [ + "curl -s -o .drone.env %s" % path_to_drone_env, + "curl -s -o check_web_cache.sh %s" % path_to_check_script, + ], + } - return [ - { - "name": "get-drone-env-and-check-script", - "image": OC_UBUNTU, - "commands": [ - "curl -s -o .drone.env %s" % path_to_drone_env, - "curl -s -o check_web_cache.sh %s" % path_to_check_script, - ], - }, - { - "name": "check-for-web-cache", - "image": OC_UBUNTU, - "environment": { - "CACHE_ENDPOINT": { - "from_secret": "cache_public_s3_server", - }, - "CACHE_BUCKET": { - "from_secret": "cache_public_s3_bucket", - }, +def checkForWebCache(name): + return { + "name": "check-for-%s-cache" % name, + "image": OC_UBUNTU, + "environment": { + "CACHE_ENDPOINT": { + "from_secret": "cache_public_s3_server", + }, + "CACHE_BUCKET": { + "from_secret": "cache_public_s3_bucket", }, - "commands": [ - "bash check_web_cache.sh", - ], }, + "commands": [ + "bash -x check_web_cache.sh %s" % name, + ], + } + +def cloneWeb(): + return { + "name": "clone-web", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + ". ./.drone.env", + "rm -rf %s" % dirs["web"], + "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git %s" % dirs["web"], + "cd %s && git checkout $WEB_COMMITID" % dirs["web"], + ], + } + +def generateWebAcceptanceCache(ctx): + return [ + getDroneEnvAndCheckScript(ctx), + checkForWebCache("acceptance"), + cloneWeb(), { - "name": "clone-web", + "name": "install-yarn", "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, "commands": [ - ". ./.drone.env", - "rm -rf %s" % dirs["web"], - "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git %s" % dirs["web"], - "cd %s && git checkout $WEB_COMMITID" % dirs["web"], + "cd %s" % dirs["web"], + "cd tests/acceptance", + "retry -t 3 'yarn install --immutable --frozen-lockfile'", ], }, + { + "name": "zip-yarn", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + # zip the yarn deps before caching + "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), + "cd %s" % dirs["web"], + "cd tests/acceptance", + "tar -czvf %s .yarn" % dirs["acceptanceYarnZip"], + ], + }, + { + "name": "cache-yarn", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "source ./.drone.env", + # cache using the minio/mc client to the public bucket (long term bucket) + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["acceptanceYarnZip"], + ], + }, + ] + +def generateWebE2ECache(ctx): + return [ + getDroneEnvAndCheckScript(ctx), + checkForWebCache("e2e"), + cloneWeb(), + { + "name": "install-yarn", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + "cd %s" % dirs["web"], + "retry -t 3 'yarn install --immutable --frozen-lockfile'", + ], + }, + { + "name": "zip-yarn", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + # zip the yarn deps before caching + "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), + "cd %s" % dirs["web"], + "tar -czvf %s .yarn" % dirs["e2eYarnZip"], + ], + }, + { + "name": "cache-yarn", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "source ./.drone.env", + # cache using the minio/mc client to the public bucket (long term bucket) + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["e2eYarnZip"], + ], + }, + ] + +def generateWebCache(ctx): + return [ + getDroneEnvAndCheckScript(ctx), + checkForWebCache("web"), + cloneWeb(), { "name": "zip-web", "image": OC_UBUNTU, @@ -2928,48 +3017,6 @@ def generateWebCache(ctx): "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["webZip"], ], }, - { - "name": "install-yarn", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["web"], - "retry -t 3 'yarn install --immutable --frozen-lockfile'", - "cd tests/acceptance", - "retry -t 3 'yarn install --immutable --frozen-lockfile'", - ], - }, - { - "name": "zip-yarn", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - # zip the yarn deps before caching - "cd %s" % dirs["web"], - "tar -czvf %s .yarn" % dirs["e2eYarnZip"], - "cd tests/acceptance", - "tar -czvf %s .yarn" % dirs["acceptanceYarnZip"], - ], - }, - { - "name": "cache-yarn", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - # cache using the minio/mc client to the public bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["e2eYarnZip"], - "mc cp -r -a %s s3/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID" % dirs["acceptanceYarnZip"], - ], - }, - { - "name": "list-web-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc ls --recursive s3/$CACHE_BUCKET/ocis/web-test-runner", - ], - }, ] def restoreWebCache(): diff --git a/tests/config/drone/check_web_cache.sh b/tests/config/drone/check_web_cache.sh index 0226db5001..875be723d2 100644 --- a/tests/config/drone/check_web_cache.sh +++ b/tests/config/drone/check_web_cache.sh @@ -1,18 +1,25 @@ #!/bin/bash source .drone.env +# if no $1 is supplied end the script +# Can be web, acceptance or e2e +if [ -z "$1" ]; then + echo "No cache item is supplied." + exit 1 +fi + echo "Checking web version - $WEB_COMMITID in cache" -URL="$CACHE_ENDPOINT/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/web.tar.gz" +URL="$CACHE_ENDPOINT/$CACHE_BUCKET/ocis/web-test-runner/$WEB_COMMITID/$1.tar.gz" echo "Checking for the web cache at '$URL'." if curl --output /dev/null --silent --head --fail "$URL" then - echo "Web with commit id $WEB_COMMITID already available in cache" + echo "$1 cache with commit id $WEB_COMMITID already available." # https://discourse.drone.io/t/how-to-exit-a-pipeline-early-without-failing/3951 # exit a Pipeline early without failing exit 78 else - echo "Web with $WEB_COMMITID not available in cache" + echo "$1 cache with commit id $WEB_COMMITID was not available." fi