mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-02-25 19:47:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Use global cache for web and core repos usage

Browse Source 
Signed-off-by: Kiran Parajuli <kiranparajuli589@gmail.com>
This commit is contained in:
Kiran Parajuli
2022-08-01 17:55:17 +05:45
parent 1504aeb891
commit fa32f76f5d
5 changed files with 262 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
.drone.star
View File

@@ -41,9 +41,9 @@ DEFAULT_PHP_VERSION = "7.4"
DEFAULT_NODEJS_VERSION = "14"
dirs = {
# relative paths from /drone/src
"core": "oc10/testrunner",
"testing_app": "oc10/testing",
"core": "/drone/src/oc10/testrunner",
"testing_app": "/drone/src/oc10/testing",
"web": "/drone/src/webTestRunner",
}
# configuration
@@ -158,6 +158,19 @@ pipelineVolumeGo = \
"temp": {},
}
# minio mc environment variables
MINIO_MC_ENV = {
"MC_HOST": {
"from_secret": "cache_s3_endpoint",
},
"AWS_ACCESS_KEY_ID": {
"from_secret": "cache_s3_access_key",
},
"AWS_SECRET_ACCESS_KEY": {
"from_secret": "cache_s3_secret_key",
},
}
def pipelineDependsOn(pipeline, dependant_pipelines):
if "depends_on" in pipeline.keys():
pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines)
@@ -200,7 +213,7 @@ def main(ctx):
test_pipelines = \
cancelPreviousBuilds() + \
buildCacheWeb(ctx) + \
buildCacheWeb() + \
yarnCache(ctx) + \
[buildOcisBinaryForTesting(ctx)] + \
cacheCoreReposForTesting(ctx) + \
@@ -218,24 +231,18 @@ def main(ctx):
docs(),
]
test_pipelines.append(
pipelineDependsOn(
purgeBuildArtifactCache(ctx, "tests-yarn"),
testPipelines(ctx),
),
)
test_pipelines.append(
pipelineDependsOn(
purgeBuildArtifactCache(ctx, "ocis-binary-amd64"),
testPipelines(ctx),
),
)
test_pipelines.append(
pipelineDependsOn(
purgeBuildArtifactCache(ctx, "testrunner"),
testPipelines(ctx),
),
)
test_pipelines.append(
pipelineDependsOn(
purgeBuildArtifactCache(ctx, "testing_app"),
testPipelines(ctx),
),
)
pipelines = test_pipelines + build_release_pipelines + build_release_helpers
@@ -263,14 +270,12 @@ def main(ctx):
pipelineSanityChecks(ctx, pipelines)
return pipelines
def buildCacheWeb(ctx):
def buildCacheWeb():
return [{
"kind": "pipeline",
"type": "docker",
"name": "cache-web",
"steps": skipIfUnchanged(ctx, "cache") +
installWebTestRunner() +
rebuildBuildArtifactCache(ctx, "web-dist", "webTestRunner"),
"steps": performWebCache(),
"trigger": {
"ref": [
"refs/heads/master",
@@ -286,10 +291,10 @@ def yarnCache(ctx):
"type": "docker",
"name": "cache-yarn",
"steps": skipIfUnchanged(ctx, "cache") +
restoreBuildArtifactCache(ctx, "web-dist", "webTestRunner") +
restoreWebCache() +
yarnInstallUITests() +
rebuildBuildArtifactCache(ctx, "tests-yarn", "webTestRunner/tests/acceptance/.yarn"),
"depends_on": getPipelineNames(buildCacheWeb(ctx)),
rebuildBuildArtifactCache(ctx, "tests-yarn", "%s/tests/acceptance/.yarn" % dirs["web"]),
"depends_on": getPipelineNames(buildCacheWeb()),
"trigger": {
"ref": [
"refs/heads/master",
@@ -298,23 +303,13 @@ def yarnCache(ctx):
},
}]
def installWebTestRunner():
return [{
"name": "install-web-test-runner",
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
"commands": [
". /drone/src/.drone.env",
"git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git webTestRunner",
],
}]
def yarnInstallUITests():
return [{
"name": "yarn-install",
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
"commands": [
". /drone/src/.drone.env",
"cd webTestRunner",
"cd %s" % dirs["web"],
"git checkout $WEB_COMMITID",
"cd tests/acceptance/",
"retry -t 3 'yarn install --immutable'",
@@ -493,10 +488,7 @@ def cacheCoreReposForTesting(ctx):
"os": "linux",
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + # skip for those pipelines where core repos are not needed
cloneCoreRepos() +
rebuildBuildArtifactCache(ctx, "testrunner", dirs["core"]) +
rebuildBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]),
"steps": performCoreReposCache(),
"trigger": {
"ref": [
"refs/heads/master",
@@ -612,8 +604,8 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4):
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty) +
restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) +
restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) +
restoreCoreCache() +
restoreTestingAppCache() +
[
{
"name": "localApiTests-%s-%s" % (suite, storage),
@@ -621,10 +613,10 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4):
"environment": {
"TEST_WITH_GRAPH_API": "true",
"PATH_TO_OCIS": "/drone/src",
"PATH_TO_CORE": "/drone/src/%s" % dirs["core"],
"PATH_TO_CORE": dirs["core"],
"TEST_SERVER_URL": "https://ocis-server:9200",
"OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""),
"SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"],
"SKELETON_DIR": dirs["testing_app"],
"OCIS_SKELETON_STRATEGY": "%s" % ("copy" if storage == "owncloud" else "upload"),
"TEST_OCIS": "true",
"SEND_SCENARIO_LINE_REFERENCES": "true",
@@ -663,17 +655,20 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
"os": "linux",
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty, []) + [
{
"name": "cs3ApiTests-%s" % (storage),
"image": OC_CS3_API_VALIDATOR,
"environment": {},
"commands": [
"/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=ocis-server:9142",
],
},
] + failEarly(ctx, early_fail),
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty, []) +
[
{
"name": "cs3ApiTests-%s" % (storage),
"image": OC_CS3_API_VALIDATOR,
"environment": {},
"commands": [
"/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=ocis-server:9142",
],
},
] +
failEarly(ctx, early_fail),
"depends_on": getPipelineNames([buildOcisBinaryForTesting(ctx)]),
"trigger": {
"ref": [
@@ -699,8 +694,8 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty) +
restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) +
restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) +
restoreCoreCache() +
restoreTestingAppCache() +
[
{
"name": "oC10ApiTests-%s-storage-%s" % (storage, part_number),
@@ -708,10 +703,10 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac
"environment": {
"TEST_WITH_GRAPH_API": "true",
"PATH_TO_OCIS": "/drone/src",
"PATH_TO_CORE": "/drone/src/%s" % dirs["core"],
"PATH_TO_CORE": dirs["core"],
"TEST_SERVER_URL": "https://ocis-server:9200",
"OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""),
"SKELETON_DIR": "/drone/src/%s/data/apiSkeleton" % dirs["testing_app"],
"SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"],
"OCIS_SKELETON_STRATEGY": "%s" % ("copy" if storage == "owncloud" else "upload"),
"TEST_OCIS": "true",
"SEND_SCENARIO_LINE_REFERENCES": "true",
@@ -723,7 +718,7 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac
"UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
},
"commands": [
"make -C /drone/src/%s test-acceptance-api" % dirs["core"],
"make -C %s test-acceptance-api" % dirs["core"],
],
},
] + failEarly(ctx, early_fail),
@@ -811,10 +806,14 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
"os": "linux",
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") + restoreBuildArtifactCache(ctx, "web-dist", "webTestRunner") +
restoreBuildArtifactCache(ctx, "tests-yarn", "webTestRunner/tests/acceptance/.yarn") + yarnInstallUITests() +
ocisServer(storage, accounts_hash_difficulty) + waitForSeleniumService() + waitForMiddlewareService() +
restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) +
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
restoreWebCache() +
restoreBuildArtifactCache(ctx, "tests-yarn", "%s/tests/acceptance/.yarn" % dirs["web"]) +
yarnInstallUITests() +
ocisServer(storage, accounts_hash_difficulty) +
waitForSeleniumService() + waitForMiddlewareService() +
restoreTestingAppCache() +
[
{
"name": "webUITests",
@@ -824,7 +823,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
"BACKEND_HOST": "https://ocis-server:9200",
"RUN_ON_OCIS": "true",
"OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data",
"TESTING_DATA_DIR": "/drone/src/%s" % dirs["testing_app"],
"TESTING_DATA_DIR": dirs["testing_app"],
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
"TEST_TAGS": finalFilterTags,
"LOCAL_UPLOAD_DIR": "/uploads",
@@ -835,7 +834,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
"MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
"cd webTestRunner/tests/acceptance",
"cd %s/tests/acceptance" % dirs["web"],
"./run.sh",
],
},
@@ -993,42 +992,46 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4):
"os": "linux",
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty) + waitForSeleniumService() + waitForMiddlewareService() + [
{
"name": "WebUIAcceptanceTests",
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
"environment": {
"SERVER_HOST": "https://ocis-server:9200",
"BACKEND_HOST": "https://ocis-server:9200",
"RUN_ON_OCIS": "true",
"OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data",
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
"TEST_TAGS": "not @skipOnOCIS and not @skip",
"LOCAL_UPLOAD_DIR": "/uploads",
"NODE_TLS_REJECT_UNAUTHORIZED": 0,
"WEB_PATH": "/srv/app/web",
"FEATURE_PATH": "/drone/src/services/settings/ui/tests/acceptance/features",
"MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
". /drone/src/.drone.env",
# we need to have Web around for some general step definitions (eg. how to log in)
"git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git /srv/app/web",
"cd /srv/app/web",
"git checkout $WEB_COMMITID",
# TODO: settings/package.json has all the acceptance test dependencies
# they shouldn't be needed since we could also use them from web:/tests/acceptance/package.json
"cd /drone/src/services/settings",
"retry -t 3 'yarn install --immutable'",
"make test-acceptance-webui",
],
"volumes": [{
"name": "uploads",
"path": "/uploads",
}],
},
] + failEarly(ctx, early_fail),
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
ocisServer(storage, accounts_hash_difficulty) +
waitForSeleniumService() +
waitForMiddlewareService() +
[
{
"name": "WebUIAcceptanceTests",
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
"environment": {
"SERVER_HOST": "https://ocis-server:9200",
"BACKEND_HOST": "https://ocis-server:9200",
"RUN_ON_OCIS": "true",
"OCIS_REVA_DATA_ROOT": "/srv/app/tmp/ocis/owncloud/data",
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
"TEST_TAGS": "not @skipOnOCIS and not @skip",
"LOCAL_UPLOAD_DIR": "/uploads",
"NODE_TLS_REJECT_UNAUTHORIZED": 0,
"WEB_PATH": "/srv/app/web",
"FEATURE_PATH": "/drone/src/services/settings/ui/tests/acceptance/features",
"MIDDLEWARE_HOST": "http://middleware:3000",
},
"commands": [
". /drone/src/.drone.env",
# we need to have Web around for some general step definitions (eg. how to log in)
"git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/owncloud/web.git /srv/app/web",
"cd /srv/app/web",
"git checkout $WEB_COMMITID",
# TODO: settings/package.json has all the acceptance test dependencies
# they shouldn't be needed since we could also use them from web:/tests/acceptance/package.json
"cd /drone/src/services/settings",
"retry -t 3 'yarn install --immutable'",
"make test-acceptance-webui",
],
"volumes": [{
"name": "uploads",
"path": "/uploads",
}],
},
] + failEarly(ctx, early_fail),
"services": [
{
"name": "redis",
@@ -1949,23 +1952,6 @@ def waitForMiddlewareService():
],
}]
def cloneCoreRepos():
return [
{
"name": "clone-core-repos",
"image": OC_CI_ALPINE,
"commands": [
"source .drone.env",
"git clone -b master --depth=1 https://github.com/owncloud/testing.git %s" % dirs["testing_app"],
"ls -la %s" % dirs["testing_app"],
"git clone -b $CORE_BRANCH --single-branch --no-tags https://github.com/owncloud/core.git %s" % dirs["core"],
"ls -la %s" % dirs["core"],
"cd %s" % dirs["core"],
"git checkout $CORE_COMMITID",
],
},
]
def redis():
return [
{
@@ -2443,8 +2429,8 @@ def parallelDeployAcceptancePipeline(ctx):
},
"steps": skipIfUnchanged(ctx, "acceptance-tests") +
restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
restoreBuildArtifactCache(ctx, "testrunner", dirs["core"]) +
restoreBuildArtifactCache(ctx, "testing_app", dirs["testing_app"]) +
restoreCoreCache() +
restoreTestingAppCache() +
copyConfigs() +
parallelDeploymentOC10Server() +
owncloudLog() +
@@ -2496,8 +2482,8 @@ def parallelAcceptance(env):
"REVA_LDAP_BASE_DN": "dc=owncloud,dc=com",
"REVA_LDAP_HOSTNAME": "openldap",
"REVA_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
"SKELETON_DIR": "/drone/src/%s/data/apiSkeleton" % dirs["testing_app"],
"PATH_TO_CORE": "/drone/src/%s" % dirs["core"],
"SKELETON_DIR": "%s/data/apiSkeleton" % dirs["testing_app"],
"PATH_TO_CORE": dirs["core"],
"OCIS_REVA_DATA_ROOT": "/mnt/data/",
"EXPECTED_FAILURES_FILE": "/drone/src/tests/parallelDeployAcceptance/expected-failures-API.md",
"OCIS_SKELETON_STRATEGY": "copy",
@@ -2799,3 +2785,84 @@ def setupForLitmus():
"cat .env",
],
}]
def performWebCache():
return [
{
"name": "clone-web",
"image": OC_UBUNTU,
"commands": [
"bash -x ./tests/config/drone/clone_web.sh",
],
},
{
"name": "cache-web",
"image": MINIO_MC,
"environment": MINIO_MC_ENV,
"commands": [
"bash -x ./tests/config/drone/cache_web.sh",
],
"depends_on": ["clone-web"],
},
]
def performCoreReposCache():
return [
{
"name": "clone-core-repos",
"image": OC_UBUNTU,
"commands": [
"bash -x ./tests/config/drone/clone_core_repos.sh",
],
},
{
"name": "cache-core-repos",
"image": MINIO_MC,
"environment": MINIO_MC_ENV,
"commands": [
"bash -x ./tests/config/drone/cache_core_repos.sh",
],
"depends_on": [
"clone-core-repos",
],
},
]
def restoreCoreCache():
return [{
"name": "retore-oc10-test-runner-cache",
"image": MINIO_MC,
"environment": MINIO_MC_ENV,
"commands": [
"source ./.drone.env",
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
"mc mirror s3/owncloud/ocis/oc10-test-runner/$CORE_COMMITID %s" % dirs["core"],
"ls -la %s" % dirs["core"],
],
}]
def restoreTestingAppCache():
return [{
"name": "retore-oc10-testing-app-cache",
"image": MINIO_MC,
"environment": MINIO_MC_ENV,
"commands": [
"source ./.drone.env",
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
"mc mirror s3/owncloud/ocis/oc10-test-runner/testingApp %s" % dirs["testing_app"],
"ls -la %s" % dirs["testing_app"],
],
}]
def restoreWebCache():
return [{
"name": "retore-web-cache",
"image": MINIO_MC,
"environment": MINIO_MC_ENV,
"commands": [
"source ./.drone.env",
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
"mc mirror s3/owncloud/ocis/web-test-runner/$WEB_COMMITID %s" % dirs["web"],
"ls -la %s" % dirs["web"],
],
}]

16
tests/config/drone/cache_core_repos.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
source .drone.env
echo "Checking core version - $CORE_COMMITID in cache"
URL="https://cache.owncloud.com/owncloud/ocis/oc10-test-runner/$CORE_COMMITID"
if curl --output /dev/null --silent --head --fail "$URL"; then
echo "Web cache for ${CORE_COMMITID} already available in cache"
exit 0
else
# cache using the minio/mc client to the 'owncloud' bucket (long term bucket)
mc alias set s3 "${MC_HOST}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
mc mirror --overwrite --remove --debug oc10TestRunner "s3/owncloud/ocis/oc10-test-runner/""${CORE_COMMITID}"
mc mirror --overwrite --remove --debug testingApp s3/owncloud/ocis/oc10-testing-app
fi

15
tests/config/drone/cache_web.sh Normal file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
source .drone.env
echo "Checking web version - $WEB_COMMITID in cache"
URL="https://cache.owncloud.com/owncloud/ocis/web-test-runner/$WEB_COMMITID"
if curl --output /dev/null --silent --head --fail "$URL"; then
echo "web cache for $WEB_COMMITID already available in cache"
exit 0
else
# cache using the minio/mc client to the 'owncloud' bucket (long term bucket)
mc alias set s3 "${MC_HOST}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
mc mirror --overwrite --remove --debug webTestRunner "s3/owncloud/ocis/web-test-runner/""${WEB_COMMITID}"
fi

25
tests/config/drone/clone_core_repos.sh Normal file
View File

@@ -0,0 +1,25 @@
#!/bin/bash
source .drone.env
echo "Checking core version - $CORE_COMMITID in cache"
URL="https://cache.owncloud.com/owncloud/ocis/oc10-test-runner/$CORE_COMMITID"
if curl --output /dev/null --silent --head --fail "$URL"; then
echo "Core cache for $CORE_COMMITID already available in cache"
exit 0
else
echo "Cache for $CORE_COMMITID not available in cache, cloning..."
# clone the core repository
git clone -b "${CORE_BRANCH}" --single-branch --no-tags https://github.com/owncloud/core.git oc10TestRunner
cd oc10TestRunner && git checkout "${CORE_COMMITID}"
ls -la .
cd ..
# clone the testing app repository
git clone -b master --single-branch --no-tags https://github.com/owncloud/testing.git testingApp
ls -la testingApp
fi

18
tests/config/drone/clone_web.sh Normal file
View File

@@ -0,0 +1,18 @@
#!/bin/bash
source .drone.env
echo "Checking web version - $WEB_COMMITID in cache"
URL="https://cache.owncloud.com/owncloud/ocis/web-test-runner/$WEB_COMMITID"
if curl --output /dev/null --silent --head --fail "$URL"; then
echo "web cache for $WEB_COMMITID already available in cache"
exit 0
else
echo "Cache for $WEB_COMMITID not available in cache, cloning..."
# clone the "owncloud/web" repository
git clone -b "${WEB_BRANCH}" --single-branch --no-tags https://github.com/owncloud/web.git webTestRunner
cd webTestRunner && git checkout "${WEB_COMMITID}"
ls -la .
fi