diff --git a/.woodpecker.star b/.woodpecker.star index aec24d9247..2030aaa41c 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -307,8 +307,8 @@ config = { "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", "OC_ASYNC_UPLOADS": True, "OC_ADD_RUN_SERVICES": "antivirus", - "STORAGE_USERS_DRIVER": "decomposed", }, + "storages": ["decomposed"], }, "multiTenancy": { "suites": [ @@ -345,16 +345,29 @@ config = { }, }, }, - "apiTests": { + "coreApiTests": { "numberOfParts": 7, "skip": False, "skipExceptParts": [], + "storages": ["posix"], }, "e2eTests": { "part": { "skip": False, "totalParts": 4, # divide and run all suites in parts (divide pipelines) - "xsuites": ["search", "app-provider", "app-provider-onlyOffice", "app-store", "keycloak", "oidc", "ocm", "a11y", "mobile-view", "navigation"], # suites to skip + # suites to skip + "xsuites": [ + "search", + "app-provider", + "app-provider-onlyOffice", + "app-store", + "keycloak", + "oidc", + "ocm", + "a11y", + "mobile-view", + "navigation", + ], }, "search": { "skip": False, @@ -439,7 +452,16 @@ CI_HTTP_PROXY_ENV = { }, } +def prefixStepCommands(pipeline, commands = [], skip_steps = []): + default_skip_steps = ["evaluate-previous-run"] + skip_steps = default_skip_steps + skip_steps + for step in pipeline["steps"]: + if "commands" in step.keys() and step["name"] not in skip_steps: + step["commands"] = commands + step["commands"] + def pipelineDependsOn(pipeline, dependant_pipelines): + if type(pipeline) == "list": + pipeline = pipeline[0] if "depends_on" in pipeline.keys(): pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines) else: @@ -482,7 +504,7 @@ def main(ctx): is_release_pr = (ctx.build.event == "pull_request" and ctx.build.sender == "openclouders" and "🎉 release" in ctx.build.title.lower()) if is_release_pr: - return [licenseCheck(ctx)] + return licenseCheck(ctx) build_release_helpers = \ readyReleaseGo() @@ -498,10 +520,10 @@ def main(ctx): codestyle(ctx) + \ checkGherkinLint(ctx) + \ checkTestSuitesInExpectedFailures(ctx) + \ - buildWebCache(ctx) + \ - cacheBrowsers(ctx) + \ + pipelinesDependsOn(buildWebCache(ctx), savePipelineNumber(ctx)) + \ + pipelinesDependsOn(cacheBrowsers(ctx), savePipelineNumber(ctx)) + \ getGoBinForTesting(ctx) + \ - buildOpencloudBinaryForTesting(ctx) + \ + pipelinesDependsOn(buildOpencloudBinaryForTesting(ctx), savePipelineNumber(ctx)) + \ checkStarlark(ctx) + \ build_release_helpers + \ testOpencloudAndUploadResults(ctx) + \ @@ -546,14 +568,56 @@ def main(ctx): ), ) + test_pipelines.append( + pipelineDependsOn( + purgePipelineInfoCache(), + testPipelines(ctx), + ), + ) + pipelines = test_pipelines + build_release_pipelines + notifyMatrix(ctx) pipelineSanityChecks(pipelines) - return pipelines + return savePipelineNumber(ctx) + pipelines + +def savePipelineNumber(ctx): + base_url = "https://raw.githubusercontent.com/%s" % repo_slug + script_link = "%s/%s/tests/config/woodpecker/upload_pipeline_info.sh" % (base_url, ctx.build.commit) + return [{ + "name": "save-pipeline-info", + "skip_clone": True, + "steps": [{ + "name": "upload-info", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "curl -s -o upload_pipeline_info.sh %s" % script_link, + "bash -x upload_pipeline_info.sh", + ], + }], + "when": [ + { + "event": ["push", "manual"], + "branch": ["main", "stable-*"], + }, + event["tag"], + event["cron"], + event["pull_request"], + ], + }] + +def evaluateWorkflowStep(): + return [{ + "name": "evaluate-previous-run", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + "node tests/config/woodpecker/evaluate_pipeline.js", + ], + }] def cachePipeline(ctx, name, steps): return { - "name": "build-%s-cache" % name, + "name": "cache-%s" % name, "skip_clone": True, "steps": steps, "when": [ @@ -578,7 +642,7 @@ def buildWebCache(ctx): ] def testOpencloudAndUploadResults(ctx): - pipeline = testOpencloud(ctx) + unit_pipeline = testOpencloud(ctx) ###################################################################### # The triggers have been disabled for now, since the govulncheck can # @@ -588,8 +652,8 @@ def testOpencloudAndUploadResults(ctx): ###################################################################### #security_scan = scanOpencloud(ctx) - #return [security_scan, pipeline, scan_result_upload] - return [pipeline] + #return [security_scan] + unit_pipeline + [scan_result_upload] + return unit_pipeline def testPipelines(ctx): pipelines = [] @@ -602,25 +666,18 @@ def testPipelines(ctx): storage = "decomposed" if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]: - pipelines.append(cs3ApiTests(ctx, storage, "default")) + pipelines += cs3ApiTests(ctx, storage, "default") if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]: - pipelines.append(wopiValidatorTests(ctx, storage, "builtin", "default")) - pipelines.append(wopiValidatorTests(ctx, storage, "cs3", "default")) + pipelines += wopiValidatorTests(ctx, storage, "builtin", "default") + pipelines += wopiValidatorTests(ctx, storage, "cs3", "default") pipelines += localApiTestPipeline(ctx) - - if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]: - pipelines += apiTests(ctx) - - enable_watch_fs = [False] - if ctx.build.event == "cron": - enable_watch_fs.append(True) - - for run_with_watch_fs_enabled in enable_watch_fs: - pipelines += e2eTestPipeline(ctx, run_with_watch_fs_enabled) + multiServiceE2ePipeline(ctx, run_with_watch_fs_enabled) + pipelines += coreApiTestPipeline(ctx) + pipelines += e2eTestPipeline(ctx) + pipelines += multiServiceE2ePipeline(ctx) if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"): - pipelines += k6LoadTests(ctx) + pipelines += pipelineDependsOn(k6LoadTests(ctx), savePipelineNumber(ctx)) return pipelines @@ -716,7 +773,7 @@ def restoreGoBinCache(): ] def testOpencloud(ctx): - steps = restoreGoBinCache() + makeGoGenerate("") + [ + steps = evaluateWorkflowStep() + restoreGoBinCache() + makeGoGenerate("") + [ { "name": "golangci-lint", "image": OC_CI_GOLANG, @@ -782,8 +839,8 @@ def testOpencloud(ctx): }, ] - return { - "name": "linting_and_unitTests", + pipeline = { + "name": "test-lint-unit", "steps": steps, "when": [ event["base"], @@ -799,6 +856,13 @@ def testOpencloud(ctx): "workspace": workspace, } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + "mkdir -p cache", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] + def scanOpencloud(ctx): steps = restoreGoBinCache() + makeGoGenerate("") + [ { @@ -830,7 +894,7 @@ def scanOpencloud(ctx): def buildOpencloudBinaryForTesting(ctx): return [{ - "name": "build_opencloud_binary_for_testing", + "name": "build-opencloud-for-testing", "steps": makeNodeGenerate("") + makeGoGenerate("") + build() + @@ -994,128 +1058,11 @@ def codestyle(ctx): return pipelines -def localApiTestPipeline(ctx): - pipelines = [] - - with_remote_php = [True] - enable_watch_fs = [False] - if ctx.build.event == "cron": - with_remote_php.append(False) - enable_watch_fs.append(True) - - storages = ["posix"] - if "[decomposed]" in ctx.build.title.lower(): - storages = ["decomposed"] - - defaults = { - "suites": {}, - "skip": False, - "extraTestEnvironment": {}, - "extraServerEnvironment": {}, - "storages": storages, - "accounts_hash_difficulty": 4, - "emailNeeded": False, - "antivirusNeeded": False, - "tikaNeeded": False, - "federationServer": False, - "collaborationServiceNeeded": False, - "extraCollaborationEnvironment": {}, - "withRemotePhp": with_remote_php, - "enableWatchFs": enable_watch_fs, - "ldapNeeded": False, - "generateVirusFiles": False, - } - - if "localApiTests" in config: - for name, matrix in config["localApiTests"].items(): - if "skip" not in matrix or not matrix["skip"]: - params = {} - for item in defaults: - params[item] = matrix[item] if item in matrix else defaults[item] - for storage in params["storages"]: - for run_with_remote_php in params["withRemotePhp"]: - for run_with_watch_fs_enabled in params["enableWatchFs"]: - pipeline = { - "name": "%s-%s%s-%s%s" % ("CLI" if name.startswith("cli") else "API", name, "-withoutRemotePhp" if not run_with_remote_php else "", "decomposed" if name.startswith("cli") else storage, "-watchfs" if run_with_watch_fs_enabled else ""), - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + - (tikaService() if params["tikaNeeded"] else []) + - (waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) + - (waitForClamavService() if params["antivirusNeeded"] else []) + - (waitForEmailService() if params["emailNeeded"] else []) + - (ldapService() if params["ldapNeeded"] else []) + - (waitForLdapService() if params["ldapNeeded"] else []) + - opencloudServer(storage, params["accounts_hash_difficulty"], extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, tika_enabled = params["tikaNeeded"], watch_fs_enabled = run_with_watch_fs_enabled) + - (opencloudServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) + - ((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) + - (openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) + - localApiTests(name, params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php, params["generateVirusFiles"]) + - logRequests(), - "services": (emailService() if params["emailNeeded"] else []) + - (clamavService() if params["antivirusNeeded"] else []) + - ((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []), - "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)), - "when": [ - event["base"], - event["cron"], - { - "event": "pull_request", - "path": { - "exclude": skipIfUnchanged(ctx, "acceptance-tests"), - }, - }, - ], - } - pipelines.append(pipeline) - return pipelines - -def localApiTests(name, suites, storage = "decomposed", extra_environment = {}, with_remote_php = False, generate_virus_files = False): - test_dir = "%s/tests/acceptance" % dirs["base"] - expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage) - - environment = { - "TEST_SERVER_URL": OC_URL, - "TEST_SERVER_FED_URL": OC_FED_URL, - "SEND_SCENARIO_LINE_REFERENCES": True, - "STORAGE_DRIVER": storage, - "BEHAT_SUITES": ",".join(suites), - "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOpencloud-%s-Storage" % storage, - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, - "WITH_REMOTE_PHP": with_remote_php, - "COLLABORATION_SERVICE_URL": "http://wopi-fakeoffice:9300", - "OC_STORAGE_PATH": "$HOME/.opencloud/storage/users", - "USE_BEARER_TOKEN": True, - } - - for item in extra_environment: - environment[item] = extra_environment[item] - - commands = [] - - # Generate EICAR virus test files if needed - if generate_virus_files: - commands.append("chmod +x %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"]) - commands.append("bash %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"]) - - # Merge expected failures - if not with_remote_php: - commands.append("cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file)) - - # Run tests - commands.append("make -C %s test-acceptance-api" % (dirs["base"])) - - return [{ - "name": "localApiTests-%s" % name, - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": environment, - "commands": commands, - }] - def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): - return { - "name": "cs3ApiTests-%s" % storage, - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + pipeline = { + "name": "test-cs3-API-%s" % storage, + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage, accounts_hash_difficulty, deploy_type = "cs3api_validator") + [ { @@ -1141,6 +1088,11 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = 4): testgroups = [ @@ -1214,10 +1166,11 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = ], }) - return { - "name": "wopiValidatorTests-%s-%s" % (wopiServerType, storage), + pipeline = { + "name": "test-wopi-validator-%s-%s" % (wopiServerType, storage), "services": fakeOffice(), - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + waitForServices("fake-office", ["fakeoffice:8080"]) + opencloudServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) + wopiServer + @@ -1255,63 +1208,14 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] -def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = False, accounts_hash_difficulty = 4, watch_fs_enabled = False): - storage = "posix" - if "[decomposed]" in ctx.build.title.lower(): - storage = "decomposed" - filterTags = "~@skipOnOpencloud-%s-Storage" % storage - test_dir = "%s/tests/acceptance" % dirs["base"] - expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage) - - return { - "name": "Core-API-Tests-%s%s-%s%s" % (part_number, "-withoutRemotePhp" if not with_remote_php else "", storage, "-watchfs" if watch_fs_enabled else ""), - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + - opencloudServer(storage, accounts_hash_difficulty, with_wrapper = True, watch_fs_enabled = watch_fs_enabled) + - [ - { - "name": "oC10ApiTests-%s" % part_number, - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": { - "TEST_SERVER_URL": OC_URL, - "OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""), - "SEND_SCENARIO_LINE_REFERENCES": True, - "STORAGE_DRIVER": storage, - "BEHAT_FILTER_TAGS": filterTags, - "DIVIDE_INTO_NUM_PARTS": number_of_parts, - "RUN_PART": part_number, - "ACCEPTANCE_TEST_TYPE": "core-api", - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, - "WITH_REMOTE_PHP": with_remote_php, - }, - "commands": [ - # merge the expected failures - "" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), - "make -C %s test-acceptance-api" % (dirs["base"]), - ], - }, - ] + - logRequests(), - "services": redisForOCStorage(storage), - "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)), - "when": [ - event["base"], - event["cron"], - { - "event": "pull_request", - "path": { - "exclude": skipIfUnchanged(ctx, "acceptance-tests"), - }, - }, - ], - } - -def apiTests(ctx): +def localApiTestPipeline(ctx): pipelines = [] - debugParts = config["apiTests"]["skipExceptParts"] - debugPartsEnabled = (len(debugParts) != 0) with_remote_php = [True] enable_watch_fs = [False] @@ -1320,19 +1224,248 @@ def apiTests(ctx): enable_watch_fs.append(True) defaults = { + "suites": {}, + "skip": False, + "extraTestEnvironment": {}, + "extraServerEnvironment": {}, + "storages": ["posix"], + "accounts_hash_difficulty": 4, + "emailNeeded": False, + "antivirusNeeded": False, + "tikaNeeded": False, + "federationServer": False, + "collaborationServiceNeeded": False, + "extraCollaborationEnvironment": {}, "withRemotePhp": with_remote_php, "enableWatchFs": enable_watch_fs, + "ldapNeeded": False, + "generateVirusFiles": False, } - for runPart in range(1, config["apiTests"]["numberOfParts"] + 1): - for run_with_remote_php in defaults["withRemotePhp"]: - for run_with_watch_fs_enabled in defaults["enableWatchFs"]: - if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts): - pipelines.append(coreApiTests(ctx, runPart, config["apiTests"]["numberOfParts"], run_with_remote_php, watch_fs_enabled = run_with_watch_fs_enabled)) + if "localApiTests" in config: + for name, matrix in config["localApiTests"].items(): + if "skip" not in matrix or not matrix["skip"]: + params = {} + for item in defaults: + params[item] = matrix[item] if item in matrix else defaults[item] + # use decomposed storage if specified in the PR title + # run CLI tests only with decomposed storage + if "[decomposed]" in ctx.build.title.lower() or name.startswith("cli"): + params["storages"] = ["decomposed"] + + for storage in params["storages"]: + for run_with_remote_php in params["withRemotePhp"]: + for run_with_watch_fs_enabled in params["enableWatchFs"]: + pipeline_name = "test-API" + if name.startswith("cli"): + pipeline_name = "test-CLI" + pipeline_name += "-%s" % name + if not run_with_remote_php: + pipeline_name += "-withoutRemotePhp" + pipeline_name += "-%s" % storage + if run_with_watch_fs_enabled: + pipeline_name += "-watchfs" + + pipeline = { + "name": pipeline_name, + "steps": evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + (tikaService() if params["tikaNeeded"] else []) + + (waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) + + (waitForClamavService() if params["antivirusNeeded"] else []) + + (waitForEmailService() if params["emailNeeded"] else []) + + (ldapService() if params["ldapNeeded"] else []) + + (waitForLdapService() if params["ldapNeeded"] else []) + + opencloudServer( + storage, + params["accounts_hash_difficulty"], + extra_server_environment = params["extraServerEnvironment"], + with_wrapper = True, + tika_enabled = params["tikaNeeded"], + watch_fs_enabled = run_with_watch_fs_enabled, + ) + + (opencloudServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) + + ((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) + + (openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) + + localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php, params["generateVirusFiles"]) + + logRequests(), + "services": (emailService() if params["emailNeeded"] else []) + + (clamavService() if params["antivirusNeeded"] else []) + + ((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []), + "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)), + "when": [ + event["base"], + event["cron"], + { + "event": "pull_request", + "path": { + "exclude": skipIfUnchanged(ctx, "acceptance-tests"), + }, + }, + ], + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) return pipelines -def e2eTestPipeline(ctx, watch_fs_enabled = False): +def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_remote_php = False, generate_virus_files = False): + test_dir = "%s/tests/acceptance" % dirs["base"] + expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage) + + environment = { + "TEST_SERVER_URL": OC_URL, + "TEST_SERVER_FED_URL": OC_FED_URL, + "SEND_SCENARIO_LINE_REFERENCES": True, + "STORAGE_DRIVER": storage, + "BEHAT_SUITES": ",".join(suites), + "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOpencloud-%s-Storage" % storage, + "EXPECTED_FAILURES_FILE": expected_failures_file, + "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, + "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, + "WITH_REMOTE_PHP": with_remote_php, + "COLLABORATION_SERVICE_URL": "http://wopi-fakeoffice:9300", + "OC_STORAGE_PATH": "$HOME/.opencloud/storage/users", + "USE_BEARER_TOKEN": True, + } + + for item in extra_environment: + environment[item] = extra_environment[item] + + commands = [] + + # Generate EICAR virus test files if needed + if generate_virus_files: + commands.append("chmod +x %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"]) + commands.append("bash %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"]) + + # Merge expected failures + if not with_remote_php: + commands.append("cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file)) + + # Run tests + commands.append("make -C %s test-acceptance-api" % (dirs["base"])) + + return [{ + "name": "api-tests", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "environment": environment, + "commands": commands, + }] + +def coreApiTestPipeline(ctx): + defaults = { + "withRemotePhp": [True], + "enableWatchFs": [False], + "storages": ["posix"], + "numberOfParts": 7, + "skipExceptParts": [], + "skip": False, + "accounts_hash_difficulty": 4, + } + + pipelines = [] + if "coreApiTests" in config: + matrix = config["coreApiTests"] + if matrix["skip"]: + return pipelines + + params = {} + for item in defaults: + params[item] = matrix[item] if item in matrix else defaults[item] + + # use decomposed storage if specified in the PR title + if "[decomposed]" in ctx.build.title.lower(): + params["storages"] = ["decomposed"] + + if ctx.build.event == "cron": + params["withRemotePhp"] = [True, False] + params["enableWatchFs"] = [True, False] + + debugParts = params["skipExceptParts"] + debugPartsEnabled = (len(debugParts) != 0) + + for storage in params["storages"]: + for runPart in range(1, params["numberOfParts"] + 1): + for run_with_remote_php in params["withRemotePhp"]: + for run_with_watch_fs_enabled in params["enableWatchFs"]: + if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts): + pipeline_name = "test-Core-API-%s" % runPart + if not run_with_remote_php: + pipeline_name += "-withoutRemotePhp" + pipeline_name += "-%s" % storage + if run_with_watch_fs_enabled: + pipeline_name += "-watchfs" + + pipeline = { + "name": pipeline_name, + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + opencloudServer( + storage, + params["accounts_hash_difficulty"], + with_wrapper = True, + watch_fs_enabled = run_with_watch_fs_enabled, + ) + + coreApiTest( + runPart, + params["numberOfParts"], + run_with_remote_php, + storage, + ) + + logRequests(), + "services": redisForOCStorage(storage), + "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)), + "when": [ + event["base"], + event["cron"], + { + "event": "pull_request", + "path": { + "exclude": skipIfUnchanged(ctx, "acceptance-tests"), + }, + }, + ], + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) + return pipelines + +def coreApiTest(part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "posix"): + filter_tags = "~@skipOnOpencloud-%s-Storage" % storage + test_dir = "%s/tests/acceptance" % dirs["base"] + expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage) + + return [{ + "name": "api-tests", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "environment": { + "TEST_SERVER_URL": OC_URL, + "OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""), + "SEND_SCENARIO_LINE_REFERENCES": True, + "STORAGE_DRIVER": storage, + "BEHAT_FILTER_TAGS": filter_tags, + "DIVIDE_INTO_NUM_PARTS": number_of_parts, + "RUN_PART": part_number, + "ACCEPTANCE_TEST_TYPE": "core-api", + "EXPECTED_FAILURES_FILE": expected_failures_file, + "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, + "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, + "WITH_REMOTE_PHP": with_remote_php, + }, + "commands": [ + # merge the expected failures + "" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), + "make -C %s test-acceptance-api" % (dirs["base"]), + ], + }] + +def e2eTestPipeline(ctx): defaults = { "skip": False, "suites": [], @@ -1340,6 +1473,8 @@ def e2eTestPipeline(ctx, watch_fs_enabled = False): "totalParts": 0, "tikaNeeded": False, "reportTracing": False, + "enableWatchFs": [False], + "storages": ["posix"], } extra_server_environment = { @@ -1372,10 +1507,6 @@ def e2eTestPipeline(ctx, watch_fs_enabled = False): if ctx.build.event == "tag": return pipelines - storage = "posix" - if "[decomposed]" in ctx.build.title.lower(): - storage = "decomposed" - for name, suite in config["e2eTests"].items(): if "skip" in suite and suite["skip"]: continue @@ -1384,6 +1515,12 @@ def e2eTestPipeline(ctx, watch_fs_enabled = False): for item in defaults: params[item] = suite[item] if item in suite else defaults[item] + if ctx.build.event == "cron": + params["enableWatchFs"] = [False, True] + + if "[decomposed]" in ctx.build.title.lower(): + params["storages"] = ["decomposed"] + e2e_args = "" if params["totalParts"] > 0: e2e_args = "--total-parts %d" % params["totalParts"] @@ -1397,61 +1534,78 @@ def e2eTestPipeline(ctx, watch_fs_enabled = False): if "with-tracing" in ctx.build.title.lower(): params["reportTracing"] = True - steps_before = \ - restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - restoreBrowsersCache() + \ - (tikaService() if params["tikaNeeded"] else []) + \ - opencloudServer(storage, extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"], watch_fs_enabled = watch_fs_enabled) + for storage in params["storages"]: + for watch_fs_enabled in params["enableWatchFs"]: + steps_before = \ + evaluateWorkflowStep() + \ + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ + restoreWebCache() + \ + restoreWebPnpmCache() + \ + restoreBrowsersCache() + \ + (tikaService() if params["tikaNeeded"] else []) + \ + opencloudServer( + storage, + extra_server_environment = extra_server_environment, + tika_enabled = params["tikaNeeded"], + watch_fs_enabled = watch_fs_enabled, + ) - step_e2e = { - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "OC_BASE_URL": OC_DOMAIN, - "HEADLESS": True, - "RETRY": "1", - "WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]), - "LOCAL_UPLOAD_DIR": "/uploads", - "PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"), - "BROWSER": "chromium", - "REPORT_TRACING": params["reportTracing"], - }, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - ], - } + step_e2e = { + "name": "e2e-tests", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "environment": { + "OC_BASE_URL": OC_DOMAIN, + "HEADLESS": True, + "RETRY": "1", + "WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]), + "LOCAL_UPLOAD_DIR": "/uploads", + "PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"), + "BROWSER": "chromium", + "REPORT_TRACING": params["reportTracing"], + }, + "commands": [ + "cd %s/tests/e2e" % dirs["web"], + ], + } - steps_after = uploadTracingResult(ctx) - - if params["totalParts"]: - for index in range(params["totalParts"]): - run_part = index + 1 - run_e2e = {} - run_e2e.update(step_e2e) - run_e2e["commands"] = [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), - ] - pipelines.append({ - "name": "e2e-tests-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""), - "steps": steps_before + [run_e2e] + steps_after, - "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "when": e2e_trigger, - }) - else: - step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) - pipelines.append({ - "name": "e2e-tests-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""), - "steps": steps_before + [step_e2e] + steps_after, - "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "when": e2e_trigger, - }) + steps_after = uploadTracingResult(ctx) + if params["totalParts"]: + for index in range(params["totalParts"]): + run_part = index + 1 + run_e2e = {} + run_e2e.update(step_e2e) + run_e2e["commands"] = [ + "cd %s/tests/e2e" % dirs["web"], + "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), + ] + pipeline = { + "name": "test-e2e-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""), + "steps": steps_before + [run_e2e] + steps_after, + "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), + "when": e2e_trigger, + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) + else: + step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) + pipeline = { + "name": "test-e2e-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""), + "steps": steps_before + [step_e2e] + steps_after, + "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), + "when": e2e_trigger, + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) return pipelines -def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): +def multiServiceE2ePipeline(ctx): pipelines = [] defaults = { @@ -1460,6 +1614,8 @@ def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): "xsuites": [], "tikaNeeded": False, "reportTracing": False, + "enableWatchFs": [False], + "storages": ["posix"], } e2e_trigger = [ @@ -1480,10 +1636,6 @@ def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): if not "full-ci" in ctx.build.title.lower() and ctx.build.event != "cron": return pipelines - storage = "posix" - if "[decomposed]" in ctx.build.title.lower(): - storage = "decomposed" - extra_server_environment = { "OC_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"], "OC_JWT_SECRET": "some-opencloud-jwt-secret", @@ -1498,9 +1650,6 @@ def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): "GRAPH_AVAILABLE_ROLES": "%s" % GRAPH_AVAILABLE_ROLES, } - if watch_fs_enabled: - extra_server_environment["STORAGE_USERS_POSIX_WATCH_FS"] = True - storage_users_environment = { "OC_CORS_ALLOW_ORIGINS": "%s,https://%s:9201" % (OC_URL, OC_SERVER_NAME), "STORAGE_USERS_JWT_SECRET": "some-opencloud-jwt-secret", @@ -1545,6 +1694,12 @@ def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): for item in defaults: params[item] = suite[item] if item in suite else defaults[item] + if ctx.build.event == "cron": + params["enableWatchFs"] = [False, True] + + if "[decomposed]" in ctx.build.title.lower(): + params["storages"] = ["decomposed"] + e2e_args = "" if params["suites"]: e2e_args = "--suites %s" % ",".join(params["suites"]) @@ -1556,38 +1711,49 @@ def multiServiceE2ePipeline(ctx, watch_fs_enabled = False): if "with-tracing" in ctx.build.title.lower(): params["reportTracing"] = True - steps = \ - restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - restoreBrowsersCache() + \ - tikaService() + \ - opencloudServer(storage, extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"]) + \ - storage_users_services + \ - [{ - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "OC_BASE_URL": OC_DOMAIN, - "HEADLESS": True, - "RETRY": "1", - "REPORT_TRACING": params["reportTracing"], - "PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"), - "BROWSER": "chromium", - }, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s" % e2e_args, - ], - }] + \ - uploadTracingResult(ctx) + for storage in params["storages"]: + for watch_fs_enabled in params["enableWatchFs"]: + if watch_fs_enabled: + extra_server_environment["STORAGE_USERS_POSIX_WATCH_FS"] = True - pipelines.append({ - "name": "e2e-tests-multi-service%s" % ("-watchfs" if watch_fs_enabled else ""), - "steps": steps, - "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "when": e2e_trigger, - }) + steps = \ + evaluateWorkflowStep() + \ + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ + restoreWebCache() + \ + restoreWebPnpmCache() + \ + restoreBrowsersCache() + \ + tikaService() + \ + opencloudServer(storage, extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"]) + \ + storage_users_services + \ + [{ + "name": "e2e-tests", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "environment": { + "OC_BASE_URL": OC_DOMAIN, + "HEADLESS": True, + "RETRY": "1", + "REPORT_TRACING": params["reportTracing"], + "PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"), + "BROWSER": "chromium", + }, + "commands": [ + "cd %s/tests/e2e" % dirs["web"], + "bash run-e2e.sh %s" % e2e_args, + ], + }] + \ + uploadTracingResult(ctx) + + pipeline = { + "name": "test-e2e-multi-service%s" % ("-watchfs" if watch_fs_enabled else ""), + "steps": steps, + "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), + "when": e2e_trigger, + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) return pipelines def uploadTracingResult(ctx): @@ -1879,7 +2045,7 @@ def binaryRelease(ctx, arch, depends_on = []): } def licenseCheck(ctx): - return { + return [{ "name": "check-licenses", "steps": restoreGoBinCache() + [ { @@ -1943,7 +2109,7 @@ def licenseCheck(ctx): event["tag"], ], "workspace": workspace, - } + }] def readyReleaseGo(): return [{ @@ -2206,9 +2372,9 @@ def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depend "%s/bin/ocwrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocWrapper"], dirs["opencloudBin"], environment["OC_URL"]), ] else: - server_commands += [ + server_commands.append( "%s server" % dirs["opencloudBin"], - ] + ) wait_for_opencloud = { "name": "wait-for-%s" % container_name, @@ -2514,6 +2680,9 @@ def purgeOpencloudWebBuildCache(ctx): def purgeGoBinCache(ctx): return purgeCache("purge_go_bin_cache", "dev/opencloud/go-bin", 14) +def purgePipelineInfoCache(): + return purgeCache("purge_pipeline_info_cache", "public/opencloud/pipelines", 14) + def pipelineSanityChecks(pipelines): """pipelineSanityChecks helps the CI developers to find errors before running it @@ -2581,10 +2750,8 @@ def pipelineSanityChecks(pipelines): print(" %sx\t%s" % (images[image], image)) def litmus(ctx, storage): - pipelines = [] - if not config["litmus"]: - return pipelines + return [] environment = { "LITMUS_PASSWORD": "admin", @@ -2592,11 +2759,12 @@ def litmus(ctx, storage): "TESTS": "basic copymove props http", } - litmusCommand = "/usr/local/bin/litmus-wrapper" + litmus_command = "/usr/local/bin/litmus-wrapper" - result = { - "name": "litmus", - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + pipeline = { + "name": "test-litmus", + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage) + setupForLitmus() + [ @@ -2607,7 +2775,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/webdav"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2617,7 +2785,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/dav/files/admin"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2627,7 +2795,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/dav/files/admin/Shares/new_folder/"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2637,7 +2805,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/webdav/Shares/new_folder/"' % OC_URL, - litmusCommand, + litmus_command, ], }, # { @@ -2651,7 +2819,7 @@ def litmus(ctx, storage): # "commands": [ # "source .env", # "export LITMUS_URL='%s/remote.php/dav/public-files/'$PUBLIC_TOKEN" % OCIS_URL, - # litmusCommand, + # litmus_command, # ], # }, { @@ -2661,7 +2829,7 @@ def litmus(ctx, storage): "commands": [ "source .env", "export LITMUS_URL='%s/remote.php/dav/spaces/'$SPACE_ID" % OC_URL, - litmusCommand, + litmus_command, ], }, ], @@ -2678,9 +2846,12 @@ def litmus(ctx, storage): }, ], } - pipelines.append(result) - return pipelines + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def setupForLitmus(): return [{ @@ -3134,10 +3305,10 @@ def k6LoadTests(ctx): if "k6-test" in ctx.build.title.lower(): event_array.append("pull_request") - return [{ - "name": "k6-load-test", + pipeline = { + "name": "test-k6-load", "skip_clone": True, - "steps": [ + "steps": evaluateWorkflowStep() + [ { "name": "k6-load-test", "image": OC_CI_ALPINE, @@ -3182,7 +3353,12 @@ def k6LoadTests(ctx): "event": event_array, }, ], - }] + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def waitForServices(name, services = []): services = ",".join(services) diff --git a/tests/config/woodpecker/evaluate_pipeline.js b/tests/config/woodpecker/evaluate_pipeline.js new file mode 100644 index 0000000000..87f0446597 --- /dev/null +++ b/tests/config/woodpecker/evaluate_pipeline.js @@ -0,0 +1,102 @@ +const fs = require("fs"); + +const CI_REPO_NAME = process.env.CI_REPO_NAME; +const CI_COMMIT_SHA = process.env.CI_COMMIT_SHA; +const CI_WORKFLOW_NAME = process.env.CI_WORKFLOW_NAME; +const CI_PIPELINE_EVENT = process.env.CI_PIPELINE_EVENT; + +const opencloudBuildWorkflow = "build-opencloud-for-testing"; +const webCacheWorkflows = ["cache-web", "cache-web-pnpm", "cache-browsers"]; + +const INFO_URL = `https://s3.ci.opencloud.eu/public/${CI_REPO_NAME}/pipelines/${CI_COMMIT_SHA}-${CI_PIPELINE_EVENT}/pipeline_info.json`; + +function getWorkflowNames(workflows) { + const allWorkflows = []; + for (const workflow of workflows) { + allWorkflows.push(workflow.name); + } + return allWorkflows; +} + +function getFailedWorkflows(workflows) { + const failedWorkflows = []; + for (const workflow of workflows) { + if (workflow.state !== "success") { + failedWorkflows.push(workflow.name); + } + } + return failedWorkflows; +} + +function hasFailingTestWorkflow(failedWorkflows) { + for (const workflowName of failedWorkflows) { + if (workflowName.startsWith("test-")) { + return true; + } + } + return false; +} + +function hasFailingE2eTestWorkflow(failedWorkflows) { + for (const workflowName of failedWorkflows) { + if (workflowName.startsWith("test-e2e-")) { + return true; + } + } + return false; +} + +async function main() { + const infoResponse = await fetch(INFO_URL); + if (infoResponse.status === 404) { + console.log("[INFO] No matching previous pipeline found. Continue..."); + process.exit(0); + } else if (!infoResponse.ok) { + console.error( + "[ERROR] Failed to fetch previous pipeline info:" + + `\n URL: ${INFO_URL}\n Status: ${infoResponse.status}` + ); + process.exit(1); + } + const info = await infoResponse.json(); + console.log(info); + + if (info.status === "success") { + console.log( + "[INFO] All workflows passed in previous pipeline. Full restart. Continue..." + ); + process.exit(0); + } + + const allWorkflows = getWorkflowNames(info.workflows); + const failedWorkflows = getFailedWorkflows(info.workflows); + + // NOTE: implement for test pipelines only for now + // // run the build workflow if any test workflow has failed + // if ( + // CI_WORKFLOW_NAME === opencloudBuildWorkflow && + // hasFailingTestWorkflow(failedWorkflows) + // ) { + // process.exit(0); + // } + + // // run the web cache workflows if any e2e test workflow has failed + // if ( + // webCacheWorkflows.includes(CI_WORKFLOW_NAME) && + // hasFailingE2eTestWorkflow(failedWorkflows) + // ) { + // process.exit(0); + // } + + if (!allWorkflows.includes(CI_WORKFLOW_NAME)) { + process.exit(0); + } + if (!failedWorkflows.includes(CI_WORKFLOW_NAME)) { + console.log("[INFO] Workflow passed in previous pipeline. Skip..."); + fs.appendFileSync(".woodpecker.env", "SKIP_WORKFLOW=true\n"); + process.exit(0); + } + console.log("[INFO] Restarting previously failed workflow. Continue..."); +} + +main(); diff --git a/tests/config/woodpecker/upload_pipeline_info.sh b/tests/config/woodpecker/upload_pipeline_info.sh new file mode 100644 index 0000000000..4455066c4d --- /dev/null +++ b/tests/config/woodpecker/upload_pipeline_info.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +CACHE_KEY="$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA-$CI_PIPELINE_EVENT" + +mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY + +# check previous pipeline +URL="https://s3.ci.opencloud.eu/$CACHE_KEY/prev_pipeline" +status=$(curl -s -o prev_pipeline "$URL" -w '%{http_code}') + +if [ "$status" == "200" ]; +then + source prev_pipeline + REPO_ID=$(printf '%s' "$CI_PIPELINE_URL" | sed 's|.*/repos/\([0-9]*\)/.*|\1|') + p_status=$(curl -s -o pipeline_info.json "$CI_SYSTEM_URL/api/repos/$REPO_ID/pipelines/$PREV_PIPELINE_NUMBER" -w "%{http_code}") + if [ "$p_status" != "200" ]; + then + echo -e "[ERROR] Failed to fetch previous pipeline info.\n URL: $CI_SYSTEM_URL/api/repos/$REPO_ID/pipelines/$PREV_PIPELINE_NUMBER\n Status: $p_status" + exit 1 + fi + # update previous pipeline info + mc cp -a pipeline_info.json "s3/$CACHE_KEY/" +fi + +# upload current pipeline number for the next pipeline +echo "PREV_PIPELINE_NUMBER=$CI_PIPELINE_NUMBER" > prev_pipeline +mc cp -a prev_pipeline "s3/$CACHE_KEY/"