From 97f88cfbe092d1c5d882bc24e840a3fd156e08f2 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Thu, 1 Jan 2026 14:09:36 +0545 Subject: [PATCH 1/4] ci: add pipeline info check scripts Signed-off-by: Saw-jan --- .woodpecker.star | 119 +++++++++++++----- tests/config/woodpecker/evaluate_pipeline.js | 89 +++++++++++++ .../config/woodpecker/upload_pipeline_info.sh | 27 ++++ 3 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 tests/config/woodpecker/evaluate_pipeline.js create mode 100644 tests/config/woodpecker/upload_pipeline_info.sh diff --git a/.woodpecker.star b/.woodpecker.star index 5f72a4f4b1..9e966aedbe 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -103,13 +103,13 @@ config = { "basic": { "suites": [ "apiArchiver", - "apiContract", - "apiCors", - "apiAsyncUpload", - "apiDownloads", - "apiDepthInfinity", - "apiLocks", - "apiActivities", + # "apiContract", + # "apiCors", + # "apiAsyncUpload", + # "apiDownloads", + # "apiDepthInfinity", + # "apiLocks", + # "apiActivities", ], "skip": False, }, @@ -117,7 +117,7 @@ config = { "suites": [ "apiSettings", ], - "skip": False, + "skip": True, "withRemotePhp": [True], "emailNeeded": True, "extraTestEnvironment": { @@ -135,7 +135,7 @@ config = { }, "graph": { "suites": [ - "apiGraph", + # "apiGraph", "apiServiceAvailability", # skip tests for collaborativePosix. see https://github.com/opencloud-eu/opencloud/issues/2036 #"collaborativePosix", @@ -147,38 +147,38 @@ config = { "suites": [ "apiGraphUserGroup", ], - "skip": False, + "skip": True, "withRemotePhp": [True], }, "spaces": { "suites": [ "apiSpaces", ], - "skip": False, + "skip": True, }, "spacesShares": { "suites": [ "apiSpacesShares", ], - "skip": False, + "skip": True, }, "spacesDavOperation": { "suites": [ "apiSpacesDavOperation", ], - "skip": False, + "skip": True, }, "search1": { "suites": [ "apiSearch1", ], - "skip": False, + "skip": True, }, "search2": { "suites": [ "apiSearch2", ], - "skip": False, + "skip": True, }, "sharingNg": { "suites": [ @@ -186,23 +186,23 @@ config = { "apiSharingNg1", "apiSharingNg2", ], - "skip": False, + "skip": True, }, "sharingNgShareInvitation": { "suites": [ "apiSharingNgShareInvitation", ], - "skip": False, + "skip": True, }, "sharingNgLinkShare": { "suites": [ "apiSharingNgLinkSharePermission", "apiSharingNgLinkShareRoot", ], - "skip": False, + "skip": True, }, "accountsHashDifficulty": { - "skip": False, + "skip": True, "suites": [ "apiAccountsHashDifficulty", ], @@ -212,7 +212,7 @@ config = { "suites": [ "apiNotification", ], - "skip": False, + "skip": True, "withRemotePhp": [True], "emailNeeded": True, "extraTestEnvironment": { @@ -232,7 +232,7 @@ config = { "suites": [ "apiAntivirus", ], - "skip": False, + "skip": True, "antivirusNeeded": True, "generateVirusFiles": True, "extraServerEnvironment": { @@ -248,14 +248,14 @@ config = { "suites": [ "apiSearchContent", ], - "skip": False, + "skip": True, "tikaNeeded": True, }, "ocm": { "suites": [ "apiOcm", ], - "skip": False, + "skip": True, "withRemotePhp": [True], "federationServer": True, "emailNeeded": True, @@ -281,7 +281,7 @@ config = { "suites": [ "apiCollaboration", ], - "skip": False, + "skip": True, "collaborationServiceNeeded": True, "extraServerEnvironment": { "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", @@ -291,14 +291,14 @@ config = { "suites": [ "apiAuthApp", ], - "skip": False, + "skip": True, "withRemotePhp": [True], }, "cliCommands": { "suites": [ "cliCommands", ], - "skip": False, + "skip": True, "withRemotePhp": [True], "antivirusNeeded": True, "generateVirusFiles": True, @@ -314,7 +314,7 @@ config = { "suites": [ "apiTenancy", ], - "skip": False, + "skip": True, "withRemotePhp": [True], "ldapNeeded": True, "extraTestEnvironment": { @@ -348,12 +348,12 @@ config = { "coreApiTests": { "numberOfParts": 7, "skip": False, - "skipExceptParts": [], + "skipExceptParts": [1], "storages": ["posix"], }, "e2eTests": { "part": { - "skip": False, + "skip": True, "totalParts": 4, # divide and run all suites in parts (divide pipelines) # suites to skip "xsuites": [ @@ -377,7 +377,7 @@ config = { }, "e2eMultiService": { "testSuites": { - "skip": False, + "skip": True, "suites": [ "smoke", "shares", @@ -492,6 +492,8 @@ def main(ctx): none """ + return savePipelineNumber(ctx) + test() + if ctx.build.event == "cron" and ctx.build.sender == "translation-sync": return translation_sync(ctx) @@ -566,9 +568,58 @@ def main(ctx): pipelineSanityChecks(pipelines) return pipelines +def savePipelineNumber(ctx): + base_url = "https://raw.githubusercontent.com/%s" % repo_slug + script_link = "%s/%s/tests/config/woodpecker/upload_pipeline_info.sh" % (base_url, ctx.build.commit) + return [{ + "name": "save-pipeline-info", + "skip_clone": True, + "steps": [{ + "name": "upload-info", + "image": MINIO_MC, + "environment": MINIO_MC_ENV, + "commands": [ + "curl -s -o upload_pipeline_info.sh %s" % script_link, + "bash -x upload_pipeline_info.sh", + ], + }], + "when": [ + { + "event": ["push", "manual"], + "branch": ["main", "stable-*"], + }, + event["tag"], + event["cron"], + event["pull_request"], + ], + }] + +def test(): + return [{ + "name": "test-pipeline", + "steps": [{ + "name": "get-previous-pipeline", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + "node --version", + "node tests/config/woodpecker/evaluate_pipeline.js", + ], + }], + "depends_on": ["save-pipeline-info"], + "when": [ + { + "event": ["push", "manual"], + "branch": ["main", "stable-*"], + }, + event["tag"], + event["cron"], + event["pull_request"], + ], + }] + def cachePipeline(ctx, name, steps): return { - "name": "build-%s-cache" % name, + "name": "cache-%s" % name, "steps": steps, "when": [ { @@ -837,7 +888,7 @@ def scanOpencloud(ctx): def buildOpencloudBinaryForTesting(ctx): return [{ - "name": "build_opencloud_binary_for_testing", + "name": "build-opencloud-for-testing", "steps": makeNodeGenerate("") + makeGoGenerate("") + build() + @@ -2285,9 +2336,9 @@ def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depend "%s/bin/ocwrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocWrapper"], dirs["opencloudBin"], environment["OC_URL"]), ] else: - server_commands += [ + server_commands.append( "%s server" % dirs["opencloudBin"], - ] + ) wait_for_opencloud = { "name": "wait-for-%s" % container_name, diff --git a/tests/config/woodpecker/evaluate_pipeline.js b/tests/config/woodpecker/evaluate_pipeline.js new file mode 100644 index 0000000000..59d61c7454 --- /dev/null +++ b/tests/config/woodpecker/evaluate_pipeline.js @@ -0,0 +1,89 @@ +// const INFO_URL="https://s3.ci.opencloud.eu/public/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/pipeline_info.json" +const CI_REPO_NAME = process.env.CI_REPO_NAME; +const CI_COMMIT_SHA = process.env.CI_COMMIT_SHA; +const CI_WORKFLOW_NAME = process.env.CI_WORKFLOW_NAME; + +const opencloudBuildWorkflow = "build-opencloud-for-testing"; +const webCacheWorkflows = ["cache-web", "cache-web-pnpm", "cache-browsers"]; + +console.log("[INFO] skip..."); +process.exit(78); + +// const INFO_URL = `https://s3.ci.opencloud.eu/public/${CI_REPO_NAME}/pipelines/${CI_COMMIT_SHA}/pipeline_info.json`; + +const INFO_URL = + "https://s3.ci.opencloud.eu/public/opencloud/pipelines/fcbcb3d9cc73fcd1cab49774632a92b531718a76/pipeline_info.json"; + +function getFailedWorkflows(workflows) { + const failedWorkflows = []; + for (const workflow of workflows) { + if (workflow.state !== "success") { + failedWorkflows.push(workflow.name); + } + } + return failedWorkflows; +} + +function hasFailingTestWorkflow(failedWorkflows) { + for (const workflowName of failedWorkflows) { + if (workflowName.startsWith("test-")) { + return true; + } + } + return false; +} + +function hasFailingE2eTestWorkflow(failedWorkflows) { + for (const workflowName of failedWorkflows) { + if (workflowName.startsWith("test-e2e-")) { + return true; + } + } + return false; +} + +async function main() { + const infoResponse = await fetch(INFO_URL); + console.log(infoResponse); + if (infoResponse.status === 404) { + console.log("[INFO] No matching previous pipeline found. Continue..."); + process.exit(0); + } else if (!infoResponse.ok) { + console.error( + "[ERROR] Failed to fetch previous pipeline info:" + + `\n URL: ${INFO_URL}\n Status: ${infoResponse.status}` + ); + process.exit(1); + } + const info = await infoResponse.json(); + + if (info.status === "success") { + process.exit(0); + } + + const failedWorkflows = getFailedWorkflows(info.workflows); + + // run the build workflow if any test workflow has failed + if ( + CI_WORKFLOW_NAME === opencloudBuildWorkflow && + hasFailingTestWorkflow(failedWorkflows) + ) { + process.exit(0); + } + + // run the web cache workflows if any e2e test workflow has failed + if ( + webCacheWorkflows.includes(CI_WORKFLOW_NAME) && + hasFailingE2eTestWorkflow(failedWorkflows) + ) { + process.exit(0); + } + + if (!failedWorkflows.includes(CI_WORKFLOW_NAME)) { + console.log("[INFO] Workflow passed in previous pipeline. Skip..."); + process.exit(78); + } + console.log("[INFO] Restarting previously failed workflow. Continue..."); +} + +main(); diff --git a/tests/config/woodpecker/upload_pipeline_info.sh b/tests/config/woodpecker/upload_pipeline_info.sh new file mode 100644 index 0000000000..05521a2d61 --- /dev/null +++ b/tests/config/woodpecker/upload_pipeline_info.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e +CI_COMMIT_SHA=fcbcb3d9cc73fcd1cab49774632a92b531718a76 +mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY + +# check previous pipeline +URL="https://s3.ci.opencloud.eu/$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/prev_pipeline" +status=$(curl -s -o prev_pipeline "$URL" -w '%{http_code}') + +if [ "$status" == "200" ]; +then + source prev_pipeline + REPO_ID=$(printf '%s' "$CI_PIPELINE_URL" | sed 's|.*/repos/\([0-9]*\)/.*|\1|') + p_status=$(curl -s -o pipeline_info.json "$CI_SYSTEM_URL/api/repos/$REPO_ID/pipelines/$PREV_PIPELINE_NUMBER" -w "%{http_code}") + if [ "$p_status" != "200" ]; + then + echo -e "[ERROR] Failed to fetch previous pipeline info.\n URL: $CI_SYSTEM_URL/api/repos/$REPO_ID/pipelines/$PREV_PIPELINE_NUMBER\n Status: $p_status" + exit 1 + fi + # update previous pipeline info + mc cp -a pipeline_info.json "s3/$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/" +fi + +# upload current pipeline number for the next pipeline +echo "PREV_PIPELINE_NUMBER=$CI_PIPELINE_NUMBER" > prev_pipeline +mc cp -a prev_pipeline "s3/$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/" \ No newline at end of file From 2b9c65dad6ccd8f52343a2d54fa865ecd9b41053 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Fri, 2 Jan 2026 12:59:59 +0545 Subject: [PATCH 2/4] ci: prefix test pipelines with test Signed-off-by: Saw-jan --- .woodpecker.star | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.woodpecker.star b/.woodpecker.star index 9e966aedbe..0da9265734 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -841,7 +841,7 @@ def testOpencloud(ctx): ] return { - "name": "linting_and_unitTests", + "name": "test-lint-unit", "steps": steps, "when": [ event["base"], @@ -1054,7 +1054,7 @@ def codestyle(ctx): def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): return { - "name": "cs3ApiTests-%s" % storage, + "name": "test-cs3-API-%s" % storage, "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage, accounts_hash_difficulty, deploy_type = "cs3api_validator") + [ @@ -1155,7 +1155,7 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = }) return { - "name": "wopiValidatorTests-%s-%s" % (wopiServerType, storage), + "name": "test-wopi-validator-%s-%s" % (wopiServerType, storage), "services": fakeOffice(), "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + waitForServices("fake-office", ["fakeoffice:8080"]) + @@ -1239,9 +1239,9 @@ def localApiTestPipeline(ctx): for storage in params["storages"]: for run_with_remote_php in params["withRemotePhp"]: for run_with_watch_fs_enabled in params["enableWatchFs"]: - pipeline_name = "API" + pipeline_name = "test-API" if name.startswith("cli"): - pipeline_name = "CLI" + pipeline_name = "test-CLI" pipeline_name += "-%s" % name if not run_with_remote_php: pipeline_name += "-withoutRemotePhp" @@ -1370,7 +1370,7 @@ def coreApiTestPipeline(ctx): for run_with_remote_php in params["withRemotePhp"]: for run_with_watch_fs_enabled in params["enableWatchFs"]: if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts): - pipeline_name = "Core-API-%s" % runPart + pipeline_name = "test-Core-API-%s" % runPart if not run_with_remote_php: pipeline_name += "-withoutRemotePhp" pipeline_name += "-%s" % storage @@ -1552,7 +1552,7 @@ def e2eTestPipeline(ctx): "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), ] pipelines.append({ - "name": "e2e-tests-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""), + "name": "test-e2e-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""), "steps": steps_before + [run_e2e] + steps_after, "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), "when": e2e_trigger, @@ -1560,7 +1560,7 @@ def e2eTestPipeline(ctx): else: step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) pipelines.append({ - "name": "e2e-tests-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""), + "name": "test-e2e-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""), "steps": steps_before + [step_e2e] + steps_after, "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), "when": e2e_trigger, @@ -2725,7 +2725,7 @@ def litmus(ctx, storage): litmusCommand = "/usr/local/bin/litmus-wrapper" result = { - "name": "litmus", + "name": "test-litmus", "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage) + setupForLitmus() + From 2e2ac84c43ca64ed465b6a1480d3860c002bacff Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Fri, 2 Jan 2026 14:03:34 +0545 Subject: [PATCH 3/4] ci: implement skip-on-pass for test workflows Signed-off-by: Saw-jan --- .woodpecker.star | 242 +++++++++++------- tests/config/woodpecker/evaluate_pipeline.js | 48 ++-- .../config/woodpecker/upload_pipeline_info.sh | 4 +- 3 files changed, 169 insertions(+), 125 deletions(-) diff --git a/.woodpecker.star b/.woodpecker.star index 0da9265734..0fee2b623f 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -103,13 +103,13 @@ config = { "basic": { "suites": [ "apiArchiver", - # "apiContract", - # "apiCors", - # "apiAsyncUpload", - # "apiDownloads", - # "apiDepthInfinity", - # "apiLocks", - # "apiActivities", + "apiContract", + "apiCors", + "apiAsyncUpload", + "apiDownloads", + "apiDepthInfinity", + "apiLocks", + "apiActivities", ], "skip": False, }, @@ -117,7 +117,7 @@ config = { "suites": [ "apiSettings", ], - "skip": True, + "skip": False, "withRemotePhp": [True], "emailNeeded": True, "extraTestEnvironment": { @@ -135,7 +135,7 @@ config = { }, "graph": { "suites": [ - # "apiGraph", + "apiGraph", "apiServiceAvailability", # skip tests for collaborativePosix. see https://github.com/opencloud-eu/opencloud/issues/2036 #"collaborativePosix", @@ -147,38 +147,38 @@ config = { "suites": [ "apiGraphUserGroup", ], - "skip": True, + "skip": False, "withRemotePhp": [True], }, "spaces": { "suites": [ "apiSpaces", ], - "skip": True, + "skip": False, }, "spacesShares": { "suites": [ "apiSpacesShares", ], - "skip": True, + "skip": False, }, "spacesDavOperation": { "suites": [ "apiSpacesDavOperation", ], - "skip": True, + "skip": False, }, "search1": { "suites": [ "apiSearch1", ], - "skip": True, + "skip": False, }, "search2": { "suites": [ "apiSearch2", ], - "skip": True, + "skip": False, }, "sharingNg": { "suites": [ @@ -186,23 +186,23 @@ config = { "apiSharingNg1", "apiSharingNg2", ], - "skip": True, + "skip": False, }, "sharingNgShareInvitation": { "suites": [ "apiSharingNgShareInvitation", ], - "skip": True, + "skip": False, }, "sharingNgLinkShare": { "suites": [ "apiSharingNgLinkSharePermission", "apiSharingNgLinkShareRoot", ], - "skip": True, + "skip": False, }, "accountsHashDifficulty": { - "skip": True, + "skip": False, "suites": [ "apiAccountsHashDifficulty", ], @@ -212,7 +212,7 @@ config = { "suites": [ "apiNotification", ], - "skip": True, + "skip": False, "withRemotePhp": [True], "emailNeeded": True, "extraTestEnvironment": { @@ -232,7 +232,7 @@ config = { "suites": [ "apiAntivirus", ], - "skip": True, + "skip": False, "antivirusNeeded": True, "generateVirusFiles": True, "extraServerEnvironment": { @@ -248,14 +248,14 @@ config = { "suites": [ "apiSearchContent", ], - "skip": True, + "skip": False, "tikaNeeded": True, }, "ocm": { "suites": [ "apiOcm", ], - "skip": True, + "skip": False, "withRemotePhp": [True], "federationServer": True, "emailNeeded": True, @@ -281,7 +281,7 @@ config = { "suites": [ "apiCollaboration", ], - "skip": True, + "skip": False, "collaborationServiceNeeded": True, "extraServerEnvironment": { "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", @@ -291,14 +291,14 @@ config = { "suites": [ "apiAuthApp", ], - "skip": True, + "skip": False, "withRemotePhp": [True], }, "cliCommands": { "suites": [ "cliCommands", ], - "skip": True, + "skip": False, "withRemotePhp": [True], "antivirusNeeded": True, "generateVirusFiles": True, @@ -314,7 +314,7 @@ config = { "suites": [ "apiTenancy", ], - "skip": True, + "skip": False, "withRemotePhp": [True], "ldapNeeded": True, "extraTestEnvironment": { @@ -348,12 +348,12 @@ config = { "coreApiTests": { "numberOfParts": 7, "skip": False, - "skipExceptParts": [1], + "skipExceptParts": [], "storages": ["posix"], }, "e2eTests": { "part": { - "skip": True, + "skip": False, "totalParts": 4, # divide and run all suites in parts (divide pipelines) # suites to skip "xsuites": [ @@ -377,7 +377,7 @@ config = { }, "e2eMultiService": { "testSuites": { - "skip": True, + "skip": False, "suites": [ "smoke", "shares", @@ -454,7 +454,16 @@ CI_HTTP_PROXY_ENV = { }, } +def prefixStepCommands(pipeline, commands = [], skip_steps = []): + default_skip_steps = ["evaluate-previous-run"] + skip_steps = default_skip_steps + skip_steps + for step in pipeline["steps"]: + if "commands" in step.keys() and step["name"] not in skip_steps: + step["commands"] = commands + step["commands"] + def pipelineDependsOn(pipeline, dependant_pipelines): + if type(pipeline) == "list": + pipeline = pipeline[0] if "depends_on" in pipeline.keys(): pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines) else: @@ -492,14 +501,12 @@ def main(ctx): none """ - return savePipelineNumber(ctx) + test() - if ctx.build.event == "cron" and ctx.build.sender == "translation-sync": return translation_sync(ctx) is_release_pr = (ctx.build.event == "pull_request" and ctx.build.sender == "openclouders" and "🎉 release" in ctx.build.title.lower()) if is_release_pr: - return [licenseCheck(ctx)] + return licenseCheck(ctx) build_release_helpers = \ readyReleaseGo() @@ -515,10 +522,10 @@ def main(ctx): codestyle(ctx) + \ checkGherkinLint(ctx) + \ checkTestSuitesInExpectedFailures(ctx) + \ - buildWebCache(ctx) + \ - cacheBrowsers(ctx) + \ + pipelinesDependsOn(buildWebCache(ctx), savePipelineNumber(ctx)) + \ + pipelinesDependsOn(cacheBrowsers(ctx), savePipelineNumber(ctx)) + \ getGoBinForTesting(ctx) + \ - buildOpencloudBinaryForTesting(ctx) + \ + pipelinesDependsOn(buildOpencloudBinaryForTesting(ctx), savePipelineNumber(ctx)) + \ checkStarlark(ctx) + \ build_release_helpers + \ testOpencloudAndUploadResults(ctx) + \ @@ -566,7 +573,7 @@ def main(ctx): pipelines = test_pipelines + build_release_pipelines + notifyMatrix(ctx) pipelineSanityChecks(pipelines) - return pipelines + return savePipelineNumber(ctx) + pipelines def savePipelineNumber(ctx): base_url = "https://raw.githubusercontent.com/%s" % repo_slug @@ -594,26 +601,12 @@ def savePipelineNumber(ctx): ], }] -def test(): +def evaluateWorkflowStep(): return [{ - "name": "test-pipeline", - "steps": [{ - "name": "get-previous-pipeline", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "node --version", - "node tests/config/woodpecker/evaluate_pipeline.js", - ], - }], - "depends_on": ["save-pipeline-info"], - "when": [ - { - "event": ["push", "manual"], - "branch": ["main", "stable-*"], - }, - event["tag"], - event["cron"], - event["pull_request"], + "name": "evaluate-previous-run", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + "node tests/config/woodpecker/evaluate_pipeline.js", ], }] @@ -643,7 +636,7 @@ def buildWebCache(ctx): ] def testOpencloudAndUploadResults(ctx): - pipeline = testOpencloud(ctx) + unit_pipeline = testOpencloud(ctx) ###################################################################### # The triggers have been disabled for now, since the govulncheck can # @@ -653,8 +646,8 @@ def testOpencloudAndUploadResults(ctx): ###################################################################### #security_scan = scanOpencloud(ctx) - #return [security_scan, pipeline, scan_result_upload] - return [pipeline] + #return [security_scan] + unit_pipeline + [scan_result_upload] + return unit_pipeline def testPipelines(ctx): pipelines = [] @@ -667,10 +660,10 @@ def testPipelines(ctx): storage = "decomposed" if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]: - pipelines.append(cs3ApiTests(ctx, storage, "default")) + pipelines += cs3ApiTests(ctx, storage, "default") if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]: - pipelines.append(wopiValidatorTests(ctx, storage, "builtin", "default")) - pipelines.append(wopiValidatorTests(ctx, storage, "cs3", "default")) + pipelines += wopiValidatorTests(ctx, storage, "builtin", "default") + pipelines += wopiValidatorTests(ctx, storage, "cs3", "default") pipelines += localApiTestPipeline(ctx) pipelines += coreApiTestPipeline(ctx) @@ -678,7 +671,7 @@ def testPipelines(ctx): pipelines += multiServiceE2ePipeline(ctx) if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"): - pipelines += k6LoadTests(ctx) + pipelines += pipelineDependsOn(k6LoadTests(ctx), savePipelineNumber(ctx)) return pipelines @@ -774,7 +767,7 @@ def restoreGoBinCache(): ] def testOpencloud(ctx): - steps = restoreGoBinCache() + makeGoGenerate("") + [ + steps = evaluateWorkflowStep() + restoreGoBinCache() + makeGoGenerate("") + [ { "name": "golangci-lint", "image": OC_CI_GOLANG, @@ -840,7 +833,7 @@ def testOpencloud(ctx): }, ] - return { + pipeline = { "name": "test-lint-unit", "steps": steps, "when": [ @@ -857,6 +850,12 @@ def testOpencloud(ctx): "workspace": workspace, } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] + def scanOpencloud(ctx): steps = restoreGoBinCache() + makeGoGenerate("") + [ { @@ -1053,9 +1052,10 @@ def codestyle(ctx): return pipelines def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): - return { + pipeline = { "name": "test-cs3-API-%s" % storage, - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage, accounts_hash_difficulty, deploy_type = "cs3api_validator") + [ { @@ -1081,6 +1081,11 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = 4): testgroups = [ @@ -1154,10 +1159,11 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = ], }) - return { + pipeline = { "name": "test-wopi-validator-%s-%s" % (wopiServerType, storage), "services": fakeOffice(), - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + waitForServices("fake-office", ["fakeoffice:8080"]) + opencloudServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) + wopiServer + @@ -1195,6 +1201,11 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def localApiTestPipeline(ctx): pipelines = [] @@ -1251,7 +1262,7 @@ def localApiTestPipeline(ctx): pipeline = { "name": pipeline_name, - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + (tikaService() if params["tikaNeeded"] else []) + (waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) + (waitForClamavService() if params["antivirusNeeded"] else []) + @@ -1286,6 +1297,10 @@ def localApiTestPipeline(ctx): }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) pipelines.append(pipeline) return pipelines @@ -1379,7 +1394,8 @@ def coreApiTestPipeline(ctx): pipeline = { "name": pipeline_name, - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer( storage, params["accounts_hash_difficulty"], @@ -1406,11 +1422,15 @@ def coreApiTestPipeline(ctx): }, ], } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) pipelines.append(pipeline) return pipelines def coreApiTest(part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "posix"): - filterTags = "~@skipOnOpencloud-%s-Storage" % storage + filter_tags = "~@skipOnOpencloud-%s-Storage" % storage test_dir = "%s/tests/acceptance" % dirs["base"] expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage) @@ -1422,7 +1442,7 @@ def coreApiTest(part_number = 1, number_of_parts = 1, with_remote_php = False, s "OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""), "SEND_SCENARIO_LINE_REFERENCES": True, "STORAGE_DRIVER": storage, - "BEHAT_FILTER_TAGS": filterTags, + "BEHAT_FILTER_TAGS": filter_tags, "DIVIDE_INTO_NUM_PARTS": number_of_parts, "RUN_PART": part_number, "ACCEPTANCE_TEST_TYPE": "core-api", @@ -1510,6 +1530,7 @@ def e2eTestPipeline(ctx): for storage in params["storages"]: for watch_fs_enabled in params["enableWatchFs"]: steps_before = \ + evaluateWorkflowStep() + \ restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ restoreWebCache() + \ restoreWebPnpmCache() + \ @@ -1551,20 +1572,30 @@ def e2eTestPipeline(ctx): "cd %s/tests/e2e" % dirs["web"], "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), ] - pipelines.append({ + pipeline = { "name": "test-e2e-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""), "steps": steps_before + [run_e2e] + steps_after, "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), "when": e2e_trigger, - }) + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) else: step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) - pipelines.append({ + pipeline = { "name": "test-e2e-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""), "steps": steps_before + [step_e2e] + steps_after, "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), "when": e2e_trigger, - }) + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) return pipelines def multiServiceE2ePipeline(ctx): @@ -1679,6 +1710,7 @@ def multiServiceE2ePipeline(ctx): extra_server_environment["STORAGE_USERS_POSIX_WATCH_FS"] = True steps = \ + evaluateWorkflowStep() + \ restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \ restoreWebCache() + \ restoreWebPnpmCache() + \ @@ -1704,12 +1736,17 @@ def multiServiceE2ePipeline(ctx): }] + \ uploadTracingResult(ctx) - pipelines.append({ + pipeline = { "name": "e2e-tests-multi-service%s" % ("-watchfs" if watch_fs_enabled else ""), "steps": steps, "depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)), "when": e2e_trigger, - }) + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + pipelines.append(pipeline) return pipelines def uploadTracingResult(ctx): @@ -2010,7 +2047,7 @@ def binaryRelease(ctx, arch, depends_on = []): } def licenseCheck(ctx): - return { + return [{ "name": "check-licenses", "steps": restoreGoBinCache() + [ { @@ -2074,7 +2111,7 @@ def licenseCheck(ctx): event["tag"], ], "workspace": workspace, - } + }] def readyReleaseGo(): return [{ @@ -2711,10 +2748,8 @@ def pipelineSanityChecks(pipelines): print(" %sx\t%s" % (images[image], image)) def litmus(ctx, storage): - pipelines = [] - if not config["litmus"]: - return pipelines + return [] environment = { "LITMUS_PASSWORD": "admin", @@ -2722,11 +2757,12 @@ def litmus(ctx, storage): "TESTS": "basic copymove props http", } - litmusCommand = "/usr/local/bin/litmus-wrapper" + litmus_command = "/usr/local/bin/litmus-wrapper" - result = { + pipeline = { "name": "test-litmus", - "steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + + "steps": evaluateWorkflowStep() + + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) + opencloudServer(storage) + setupForLitmus() + [ @@ -2737,7 +2773,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/webdav"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2747,7 +2783,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/dav/files/admin"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2757,7 +2793,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/dav/files/admin/Shares/new_folder/"' % OC_URL, - litmusCommand, + litmus_command, ], }, { @@ -2767,7 +2803,7 @@ def litmus(ctx, storage): "commands": [ "source .env", 'export LITMUS_URL="%s/remote.php/webdav/Shares/new_folder/"' % OC_URL, - litmusCommand, + litmus_command, ], }, # { @@ -2781,7 +2817,7 @@ def litmus(ctx, storage): # "commands": [ # "source .env", # "export LITMUS_URL='%s/remote.php/dav/public-files/'$PUBLIC_TOKEN" % OCIS_URL, - # litmusCommand, + # litmus_command, # ], # }, { @@ -2791,7 +2827,7 @@ def litmus(ctx, storage): "commands": [ "source .env", "export LITMUS_URL='%s/remote.php/dav/spaces/'$SPACE_ID" % OC_URL, - litmusCommand, + litmus_command, ], }, ], @@ -2808,9 +2844,12 @@ def litmus(ctx, storage): }, ], } - pipelines.append(result) - return pipelines + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def setupForLitmus(): return [{ @@ -3262,10 +3301,10 @@ def k6LoadTests(ctx): if "k6-test" in ctx.build.title.lower(): event_array.append("pull_request") - return [{ - "name": "k6-load-test", + pipeline = { + "name": "test-k6-load", "skip_clone": True, - "steps": [ + "steps": evaluateWorkflowStep() + [ { "name": "k6-load-test", "image": OC_CI_ALPINE, @@ -3310,7 +3349,12 @@ def k6LoadTests(ctx): "event": event_array, }, ], - }] + } + prefixStepCommands(pipeline, [ + ". ./.woodpecker.env", + '[ "$SKIP_WORKFLOW" = "true" ] && exit 0', + ]) + return [pipeline] def waitForServices(name, services = []): services = ",".join(services) diff --git a/tests/config/woodpecker/evaluate_pipeline.js b/tests/config/woodpecker/evaluate_pipeline.js index 59d61c7454..b5e325a908 100644 --- a/tests/config/woodpecker/evaluate_pipeline.js +++ b/tests/config/woodpecker/evaluate_pipeline.js @@ -1,4 +1,5 @@ -// const INFO_URL="https://s3.ci.opencloud.eu/public/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/pipeline_info.json" +const fs = require("fs"); + const CI_REPO_NAME = process.env.CI_REPO_NAME; const CI_COMMIT_SHA = process.env.CI_COMMIT_SHA; const CI_WORKFLOW_NAME = process.env.CI_WORKFLOW_NAME; @@ -6,13 +7,7 @@ const CI_WORKFLOW_NAME = process.env.CI_WORKFLOW_NAME; const opencloudBuildWorkflow = "build-opencloud-for-testing"; const webCacheWorkflows = ["cache-web", "cache-web-pnpm", "cache-browsers"]; -console.log("[INFO] skip..."); -process.exit(78); - -// const INFO_URL = `https://s3.ci.opencloud.eu/public/${CI_REPO_NAME}/pipelines/${CI_COMMIT_SHA}/pipeline_info.json`; - -const INFO_URL = - "https://s3.ci.opencloud.eu/public/opencloud/pipelines/fcbcb3d9cc73fcd1cab49774632a92b531718a76/pipeline_info.json"; +const INFO_URL = `https://s3.ci.opencloud.eu/public/${CI_REPO_NAME}/pipelines/${CI_COMMIT_SHA}/pipeline_info.json`; function getFailedWorkflows(workflows) { const failedWorkflows = []; @@ -44,7 +39,6 @@ function hasFailingE2eTestWorkflow(failedWorkflows) { async function main() { const infoResponse = await fetch(INFO_URL); - console.log(infoResponse); if (infoResponse.status === 404) { console.log("[INFO] No matching previous pipeline found. Continue..."); process.exit(0); @@ -56,32 +50,38 @@ async function main() { process.exit(1); } const info = await infoResponse.json(); + console.log(info); if (info.status === "success") { + console.log( + "[INFO] All workflows passed in previous pipeline. Full restart. Continue..." + ); process.exit(0); } const failedWorkflows = getFailedWorkflows(info.workflows); - // run the build workflow if any test workflow has failed - if ( - CI_WORKFLOW_NAME === opencloudBuildWorkflow && - hasFailingTestWorkflow(failedWorkflows) - ) { - process.exit(0); - } + // NOTE: implement for test pipelines only for now + // // run the build workflow if any test workflow has failed + // if ( + // CI_WORKFLOW_NAME === opencloudBuildWorkflow && + // hasFailingTestWorkflow(failedWorkflows) + // ) { + // process.exit(0); + // } - // run the web cache workflows if any e2e test workflow has failed - if ( - webCacheWorkflows.includes(CI_WORKFLOW_NAME) && - hasFailingE2eTestWorkflow(failedWorkflows) - ) { - process.exit(0); - } + // // run the web cache workflows if any e2e test workflow has failed + // if ( + // webCacheWorkflows.includes(CI_WORKFLOW_NAME) && + // hasFailingE2eTestWorkflow(failedWorkflows) + // ) { + // process.exit(0); + // } if (!failedWorkflows.includes(CI_WORKFLOW_NAME)) { console.log("[INFO] Workflow passed in previous pipeline. Skip..."); - process.exit(78); + fs.appendFileSync(".woodpecker.env", "SKIP_WORKFLOW=true\n"); + process.exit(0); } console.log("[INFO] Restarting previously failed workflow. Continue..."); } diff --git a/tests/config/woodpecker/upload_pipeline_info.sh b/tests/config/woodpecker/upload_pipeline_info.sh index 05521a2d61..6662338cc3 100644 --- a/tests/config/woodpecker/upload_pipeline_info.sh +++ b/tests/config/woodpecker/upload_pipeline_info.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -CI_COMMIT_SHA=fcbcb3d9cc73fcd1cab49774632a92b531718a76 + mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY # check previous pipeline @@ -24,4 +24,4 @@ fi # upload current pipeline number for the next pipeline echo "PREV_PIPELINE_NUMBER=$CI_PIPELINE_NUMBER" > prev_pipeline -mc cp -a prev_pipeline "s3/$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/" \ No newline at end of file +mc cp -a prev_pipeline "s3/$PUBLIC_BUCKET/$CI_REPO_NAME/pipelines/$CI_COMMIT_SHA/" From 9cf10f9c986a05d32d16fe56122beb07a99bd570 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Wed, 7 Jan 2026 13:10:26 +0545 Subject: [PATCH 4/4] ci: add cache purge workflow Signed-off-by: Saw-jan --- .woodpecker.star | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.woodpecker.star b/.woodpecker.star index 0fee2b623f..d7f1d09f14 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -570,6 +570,13 @@ def main(ctx): ), ) + test_pipelines.append( + pipelineDependsOn( + purgePipelineInfoCache(), + testPipelines(ctx), + ), + ) + pipelines = test_pipelines + build_release_pipelines + notifyMatrix(ctx) pipelineSanityChecks(pipelines) @@ -2681,6 +2688,9 @@ def purgeOpencloudWebBuildCache(ctx): def purgeGoBinCache(ctx): return purgeCache("purge_go_bin_cache", "dev/opencloud/go-bin", 14) +def purgePipelineInfoCache(): + return purgeCache("purge_pipeline_info_cache", "public/opencloud/pipelines", 14) + def pipelineSanityChecks(pipelines): """pipelineSanityChecks helps the CI developers to find errors before running it