diff --git a/.drone.star b/.drone.star index d3a02c4ffe..f69f8d7018 100644 --- a/.drone.star +++ b/.drone.star @@ -41,9 +41,9 @@ DEFAULT_PHP_VERSION = "7.4" DEFAULT_NODEJS_VERSION = "14" dirs = { - # relative paths from /drone/src - "core": "oc10/testrunner", - "testing_app": "oc10/testing", + "core": "/drone/src/oc10/testrunner", + "testing_app": "/drone/src/oc10/testing", + "web": "/drone/src/webTestRunner", } # configuration @@ -158,6 +158,19 @@ pipelineVolumeGo = \ "temp": {}, } +# minio mc environment variables +MINIO_MC_ENV = { + "MC_HOST": { + "from_secret": "cache_s3_endpoint", + }, + "AWS_ACCESS_KEY_ID": { + "from_secret": "cache_s3_access_key", + }, + "AWS_SECRET_ACCESS_KEY": { + "from_secret": "cache_s3_secret_key", + }, +} + def pipelineDependsOn(pipeline, dependant_pipelines): if "depends_on" in pipeline.keys(): pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines) @@ -200,7 +213,7 @@ def main(ctx): test_pipelines = \ cancelPreviousBuilds() + \ - buildCacheWeb(ctx) + \ + buildCacheWeb() + \ yarnCache(ctx) + \ [buildOcisBinaryForTesting(ctx)] + \ cacheCoreReposForTesting(ctx) + \ @@ -218,24 +231,18 @@ def main(ctx): docs(), ] + test_pipelines.append( + pipelineDependsOn( + purgeBuildArtifactCache(ctx, "tests-yarn"), + testPipelines(ctx), + ), + ) test_pipelines.append( pipelineDependsOn( purgeBuildArtifactCache(ctx, "ocis-binary-amd64"), testPipelines(ctx), ), ) - test_pipelines.append( - pipelineDependsOn( - purgeBuildArtifactCache(ctx, "testrunner"), - testPipelines(ctx), - ), - ) - test_pipelines.append( - pipelineDependsOn( - purgeBuildArtifactCache(ctx, "testing_app"), - testPipelines(ctx), - ), - ) pipelines = test_pipelines + build_release_pipelines + build_release_helpers @@ -263,14 +270,12 @@ def main(ctx): pipelineSanityChecks(ctx, pipelines) return pipelines -def buildCacheWeb(ctx): +def buildCacheWeb(): return [{ "kind": "pipeline", "type": "docker", "name": "cache-web", - "steps": skipIfUnchanged(ctx, "cache") + - installWebTestRunner() + - rebuildBuildArtifactCache(ctx, "web-dist", "webTestRunner"), + "steps": performWebCache(), "trigger": { "ref": [ "refs/heads/master", @@ -286,10 +291,10 @@ def yarnCache(ctx): "type": "docker", "name": "cache-yarn", "steps": skipIfUnchanged(ctx, "cache") + - restoreBuildArtifactCache(ctx, "web-dist", "webTestRunner") + + restoreWebCache() + yarnInstallUITests() + - rebuildBuildArtifactCache(ctx, "tests-yarn", "webTestRunner/tests/acceptance/.yarn"), - "depends_on": getPipelineNames(buildCacheWeb(ctx)), + rebuildBuildArtifactCache(ctx, "tests-yarn", "%s/tests/acceptance/.yarn" % dirs["web"]), + "depends_on": getPipelineNames(buildCacheWeb()), "trigger": { "ref": [ "refs/heads/master", @@ -298,23 +303,13 @@ def yarnCache(ctx): }, }] -def installWebTestRunner(): - return [{ - "name": "install-web-test-runner", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - ". /drone/src/.drone.env", - "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git webTestRunner", - ], - }] - def yarnInstallUITests(): return [{ "name": "yarn-install", "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, "commands": [ ". /drone/src/.drone.env", - "cd webTestRunner", + "cd %s" % dirs["web"], "git checkout $WEB_COMMITID", "cd tests/acceptance/", "retry -t 3 'yarn install --immutable'", @@ -493,10 +488,7 @@ def cacheCoreReposForTesting(ctx): "os": "linux", "arch": "amd64", }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + # skip for those pipelines where core repos are not needed - cloneCoreRepos() + - rebuildBuildArtifactCache(ctx, "testrunner", dirs["core"]) + - rebuildBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]), + "steps": performCoreReposCache(), "trigger": { "ref": [ "refs/heads/master", @@ -612,8 +604,8 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + ocisServer(storage, accounts_hash_difficulty) + - restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) + - restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) + + restoreCoreCache() + + restoreTestingAppCache() + [ { "name": "localApiTests-%s-%s" % (suite, storage), @@ -621,10 +613,10 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): "environment": { "TEST_WITH_GRAPH_API": "true", "PATH_TO_OCIS": "/drone/src", - "PATH_TO_CORE": "/drone/src/%s" % dirs["core"], + "PATH_TO_CORE": dirs["core"], "TEST_SERVER_URL": "https://ocis-server:9200", "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""), - "SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"], + "SKELETON_DIR": dirs["testing_app"], "OCIS_SKELETON_STRATEGY": "%s" % ("copy" if storage == "owncloud" else "upload"), "TEST_OCIS": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", @@ -663,17 +655,20 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): "os": "linux", "arch": "amd64", }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - ocisServer(storage, accounts_hash_difficulty, []) + [ - { - "name": "cs3ApiTests-%s" % (storage), - "image": OC_CS3_API_VALIDATOR, - "environment": {}, - "commands": [ - "/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=ocis-server:9142", - ], - }, - ] + failEarly(ctx, early_fail), + "steps": skipIfUnchanged(ctx, "acceptance-tests") + + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + + ocisServer(storage, accounts_hash_difficulty, []) + + [ + { + "name": "cs3ApiTests-%s" % (storage), + "image": OC_CS3_API_VALIDATOR, + "environment": {}, + "commands": [ + "/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=ocis-server:9142", + ], + }, + ] + + failEarly(ctx, early_fail), "depends_on": getPipelineNames([buildOcisBinaryForTesting(ctx)]), "trigger": { "ref": [ @@ -699,8 +694,8 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + ocisServer(storage, accounts_hash_difficulty) + - restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) + - restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) + + restoreCoreCache() + + restoreTestingAppCache() + [ { "name": "oC10ApiTests-%s-storage-%s" % (storage, part_number), @@ -708,10 +703,10 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "environment": { "TEST_WITH_GRAPH_API": "true", "PATH_TO_OCIS": "/drone/src", - "PATH_TO_CORE": "/drone/src/%s" % dirs["core"], + "PATH_TO_CORE": dirs["core"], "TEST_SERVER_URL": "https://ocis-server:9200", "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""), - "SKELETON_DIR": "/drone/src/%s/data/apiSkeleton" % dirs["testing_app"], + "SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"], "OCIS_SKELETON_STRATEGY": "%s" % ("copy" if storage == "owncloud" else "upload"), "TEST_OCIS": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", @@ -723,7 +718,7 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, }, "commands": [ - "make -C /drone/src/%s test-acceptance-api" % dirs["core"], + "make -C %s test-acceptance-api" % dirs["core"], ], }, ] + failEarly(ctx, early_fail), @@ -811,10 +806,14 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1, "os": "linux", "arch": "amd64", }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + restoreBuildArtifactCache(ctx, "web-dist", "webTestRunner") + - restoreBuildArtifactCache(ctx, "tests-yarn", "webTestRunner/tests/acceptance/.yarn") + yarnInstallUITests() + - ocisServer(storage, accounts_hash_difficulty) + waitForSeleniumService() + waitForMiddlewareService() + - restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + + restoreWebCache() + + restoreBuildArtifactCache(ctx, "tests-yarn", "%s/tests/acceptance/.yarn" % dirs["web"]) + + yarnInstallUITests() + + ocisServer(storage, accounts_hash_difficulty) + + waitForSeleniumService() + waitForMiddlewareService() + + restoreTestingAppCache() + [ { "name": "webUITests", @@ -824,7 +823,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1, "BACKEND_HOST": "https://ocis-server:9200", "RUN_ON_OCIS": "true", "OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data", - "TESTING_DATA_DIR": "/drone/src/%s" % dirs["testing_app"], + "TESTING_DATA_DIR": dirs["testing_app"], "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", "TEST_TAGS": finalFilterTags, "LOCAL_UPLOAD_DIR": "/uploads", @@ -835,7 +834,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1, "MIDDLEWARE_HOST": "http://middleware:3000", }, "commands": [ - "cd webTestRunner/tests/acceptance", + "cd %s/tests/acceptance" % dirs["web"], "./run.sh", ], }, @@ -993,42 +992,46 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4): "os": "linux", "arch": "amd64", }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - ocisServer(storage, accounts_hash_difficulty) + waitForSeleniumService() + waitForMiddlewareService() + [ - { - "name": "WebUIAcceptanceTests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "SERVER_HOST": "https://ocis-server:9200", - "BACKEND_HOST": "https://ocis-server:9200", - "RUN_ON_OCIS": "true", - "OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data", - "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", - "TEST_TAGS": "not @skipOnOCIS and not @skip", - "LOCAL_UPLOAD_DIR": "/uploads", - "NODE_TLS_REJECT_UNAUTHORIZED": 0, - "WEB_PATH": "/srv/app/web", - "FEATURE_PATH": "/drone/src/services/settings/ui/tests/acceptance/features", - "MIDDLEWARE_HOST": "http://middleware:3000", - }, - "commands": [ - ". /drone/src/.drone.env", - # we need to have Web around for some general step definitions (eg. how to log in) - "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git /srv/app/web", - "cd /srv/app/web", - "git checkout $WEB_COMMITID", - # TODO: settings/package.json has all the acceptance test dependencies - # they shouldn't be needed since we could also use them from web:/tests/acceptance/package.json - "cd /drone/src/services/settings", - "retry -t 3 'yarn install --immutable'", - "make test-acceptance-webui", - ], - "volumes": [{ - "name": "uploads", - "path": "/uploads", - }], - }, - ] + failEarly(ctx, early_fail), + "steps": skipIfUnchanged(ctx, "acceptance-tests") + + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + + ocisServer(storage, accounts_hash_difficulty) + + waitForSeleniumService() + + waitForMiddlewareService() + + [ + { + "name": "WebUIAcceptanceTests", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "environment": { + "SERVER_HOST": "https://ocis-server:9200", + "BACKEND_HOST": "https://ocis-server:9200", + "RUN_ON_OCIS": "true", + "OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data", + "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", + "TEST_TAGS": "not @skipOnOCIS and not @skip", + "LOCAL_UPLOAD_DIR": "/uploads", + "NODE_TLS_REJECT_UNAUTHORIZED": 0, + "WEB_PATH": "/srv/app/web", + "FEATURE_PATH": "/drone/src/services/settings/ui/tests/acceptance/features", + "MIDDLEWARE_HOST": "http://middleware:3000", + }, + "commands": [ + ". /drone/src/.drone.env", + # we need to have Web around for some general step definitions (eg. how to log in) + "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git /srv/app/web", + "cd /srv/app/web", + "git checkout $WEB_COMMITID", + # TODO: settings/package.json has all the acceptance test dependencies + # they shouldn't be needed since we could also use them from web:/tests/acceptance/package.json + "cd /drone/src/services/settings", + "retry -t 3 'yarn install --immutable'", + "make test-acceptance-webui", + ], + "volumes": [{ + "name": "uploads", + "path": "/uploads", + }], + }, + ] + failEarly(ctx, early_fail), "services": [ { "name": "redis", @@ -1949,23 +1952,6 @@ def waitForMiddlewareService(): ], }] -def cloneCoreRepos(): - return [ - { - "name": "clone-core-repos", - "image": OC_CI_ALPINE, - "commands": [ - "source .drone.env", - "git clone -b master --depth=1 https://github.com/owncloud/testing.git %s" % dirs["testing_app"], - "ls -la %s" % dirs["testing_app"], - "git clone -b $CORE_BRANCH --single-branch --no-tags https://github.com/owncloud/core.git %s" % dirs["core"], - "ls -la %s" % dirs["core"], - "cd %s" % dirs["core"], - "git checkout $CORE_COMMITID", - ], - }, - ] - def redis(): return [ { @@ -2443,8 +2429,8 @@ def parallelDeployAcceptancePipeline(ctx): }, "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + - restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) + - restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) + + restoreCoreCache() + + restoreTestingAppCache() + copyConfigs() + parallelDeploymentOC10Server() + owncloudLog() + @@ -2496,8 +2482,8 @@ def parallelAcceptance(env): "REVA_LDAP_BASE_DN": "dc=owncloud,dc=com", "REVA_LDAP_HOSTNAME": "openldap", "REVA_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", - "SKELETON_DIR": "/drone/src/%s/data/apiSkeleton" % dirs["testing_app"], - "PATH_TO_CORE": "/drone/src/%s" % dirs["core"], + "SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"], + "PATH_TO_CORE": dirs["core"], "OCIS_REVA_DATA_ROOT": "/mnt/data/", "EXPECTED_FAILURES_FILE": "/drone/src/tests/parallelDeployAcceptance/expected-failures-API.md", "OCIS_SKELETON_STRATEGY": "copy", @@ -2799,3 +2785,84 @@ def setupForLitmus(): "cat .env", ], }] + +def performWebCache(): + return [ + { + "name": "clone-web", + "image": OC_UBUNTU, + "commands": [ + "bash -x ./tests/config/drone/clone_web.sh", + ], + }, + { + "name": "cache-web", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "bash -x ./tests/config/drone/cache_web.sh", + ], + "depends_on": ["clone-web"], + }, + ] + +def performCoreReposCache(): + return [ + { + "name": "clone-core-repos", + "image": OC_UBUNTU, + "commands": [ + "bash -x ./tests/config/drone/clone_core_repos.sh", + ], + }, + { + "name": "cache-core-repos", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "bash -x ./tests/config/drone/cache_core_repos.sh", + ], + "depends_on": [ + "clone-core-repos", + ], + }, + ] + +def restoreCoreCache(): + return [{ + "name": "retore-oc10-test-runner-cache", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "source ./.drone.env", + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc mirror s3/owncloud/ocis/oc10-test-runner/$CORE_COMMITID %s" % dirs["core"], + "ls -la %s" % dirs["core"], + ], + }] + +def restoreTestingAppCache(): + return [{ + "name": "retore-oc10-testing-app-cache", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "source ./.drone.env", + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc mirror s3/owncloud/ocis/oc10-test-runner/testingApp %s" % dirs["testing_app"], + "ls -la %s" % dirs["testing_app"], + ], + }] + +def restoreWebCache(): + return [{ + "name": "retore-web-cache", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "source ./.drone.env", + "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", + "mc mirror s3/owncloud/ocis/web-test-runner/$WEB_COMMITID %s" % dirs["web"], + "ls -la %s" % dirs["web"], + ], + }] diff --git a/tests/config/drone/cache_core_repos.sh b/tests/config/drone/cache_core_repos.sh new file mode 100644 index 0000000000..f34cfb085a --- /dev/null +++ b/tests/config/drone/cache_core_repos.sh @@ -0,0 +1,16 @@ +#!/bin/bash +source .drone.env + +echo "Checking core version - $CORE_COMMITID in cache" + +URL="https://cache.owncloud.com/owncloud/ocis/oc10-test-runner/$CORE_COMMITID" + +if curl --output /dev/null --silent --head --fail "$URL"; then + echo "Web cache for ${CORE_COMMITID} already available in cache" + exit 0 +else + # cache using the minio/mc client to the 'owncloud' bucket (long term bucket) + mc alias set s3 "${MC_HOST}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" + mc mirror --overwrite --remove --debug oc10TestRunner "s3/owncloud/ocis/oc10-test-runner/""${CORE_COMMITID}" + mc mirror --overwrite --remove --debug testingApp s3/owncloud/ocis/oc10-testing-app +fi diff --git a/tests/config/drone/cache_web.sh b/tests/config/drone/cache_web.sh new file mode 100644 index 0000000000..2ca76acb42 --- /dev/null +++ b/tests/config/drone/cache_web.sh @@ -0,0 +1,15 @@ +#!/bin/bash +source .drone.env + +echo "Checking web version - $WEB_COMMITID in cache" + +URL="https://cache.owncloud.com/owncloud/ocis/web-test-runner/$WEB_COMMITID" + +if curl --output /dev/null --silent --head --fail "$URL"; then + echo "web cache for $WEB_COMMITID already available in cache" + exit 0 +else + # cache using the minio/mc client to the 'owncloud' bucket (long term bucket) + mc alias set s3 "${MC_HOST}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" + mc mirror --overwrite --remove --debug webTestRunner "s3/owncloud/ocis/web-test-runner/""${WEB_COMMITID}" +fi diff --git a/tests/config/drone/clone_core_repos.sh b/tests/config/drone/clone_core_repos.sh new file mode 100644 index 0000000000..d1f92be6c2 --- /dev/null +++ b/tests/config/drone/clone_core_repos.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source .drone.env + +echo "Checking core version - $CORE_COMMITID in cache" + +URL="https://cache.owncloud.com/owncloud/ocis/oc10-test-runner/$CORE_COMMITID" + +if curl --output /dev/null --silent --head --fail "$URL"; then + echo "Core cache for $CORE_COMMITID already available in cache" + exit 0 +else + echo "Cache for $CORE_COMMITID not available in cache, cloning..." + + # clone the core repository + git clone -b "${CORE_BRANCH}" --single-branch --no-tags https://github.com/owncloud/core.git oc10TestRunner + + cd oc10TestRunner && git checkout "${CORE_COMMITID}" + ls -la . + cd .. + + # clone the testing app repository + git clone -b master --single-branch --no-tags https://github.com/owncloud/testing.git testingApp + + ls -la testingApp +fi diff --git a/tests/config/drone/clone_web.sh b/tests/config/drone/clone_web.sh new file mode 100644 index 0000000000..5cf74454e5 --- /dev/null +++ b/tests/config/drone/clone_web.sh @@ -0,0 +1,18 @@ +#!/bin/bash +source .drone.env + +echo "Checking web version - $WEB_COMMITID in cache" + +URL="https://cache.owncloud.com/owncloud/ocis/web-test-runner/$WEB_COMMITID" + +if curl --output /dev/null --silent --head --fail "$URL"; then + echo "web cache for $WEB_COMMITID already available in cache" + exit 0 +else + echo "Cache for $WEB_COMMITID not available in cache, cloning..." + # clone the "owncloud/web" repository + git clone -b "${WEB_BRANCH}" --single-branch --no-tags https://github.com/owncloud/web.git webTestRunner + cd webTestRunner && git checkout "${WEB_COMMITID}" + + ls -la . +fi