diff --git a/.woodpecker/build.yaml b/.woodpecker/build.yaml index 04cc468973..e778bd5f13 100644 --- a/.woodpecker/build.yaml +++ b/.woodpecker/build.yaml @@ -1,4 +1,3 @@ ---- variables: - &squish_image 'opencloudeu/squish@sha256:6eaecc218044020f49f24fd29b6bdc052e8170699a762687b10398b353e5fcda' - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' @@ -9,7 +8,25 @@ variables: from_secret: cache_s3_secret_key CACHE_BUCKET: from_secret: cache_s3_bucket - MC_HOST: "https://s3.ci.opencloud.eu" + MC_HOST: 'https://s3.ci.opencloud.eu' + +when: + - branch: + - main + - stable-* + event: + - push + - manual + - event: pull_request + evaluate: | + !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") + - event: tag + - event: cron + cron: nightly* + +workspace: + base: /woodpecker/ + path: desktop steps: - name: fix-permissions @@ -24,27 +41,10 @@ steps: - cmake %s -S .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ - ninja - name: upload-desktop-client-cache + image: *minio_image + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -a -r /woodpecker/desktop/build/bin s3/$CACHE_BUCKET/desktop-build/${CI_COMMIT_SHA}/ - mc ls --recursive s3/$CACHE_BUCKET/desktop-build - environment: - <<: *minio_environment - image: *minio_image - -when: - - branch: - - main - - stable-* - event: - - push - - manual - - event: pull_request - evaluate: | - !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") - - event: tag - - event: cron - cron: nightly* -workspace: - base: /woodpecker/ - path: desktop diff --git a/.woodpecker/cache-opencloud.yaml b/.woodpecker/cache-opencloud.yaml index 3f8eafa5e6..6c384a4264 100644 --- a/.woodpecker/cache-opencloud.yaml +++ b/.woodpecker/cache-opencloud.yaml @@ -1,4 +1,3 @@ ---- variables: - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' - &minio_environment @@ -8,63 +7,69 @@ variables: from_secret: cache_s3_secret_key CACHE_BUCKET: from_secret: cache_s3_bucket - MC_HOST: "https://s3.ci.opencloud.eu" + MC_HOST: 'https://s3.ci.opencloud.eu' + +when: + - branch: + - main + - stable-* + event: + - push + - manual + - event: pull_request + evaluate: | + !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") + - event: tag + - event: cron + cron: nightly* skip_clone: true + steps: - - commands: + - name: check-for-existing-cache + image: *minio_image + environment: + <<: *minio_environment + commands: - curl -o .woodpecker.env https://raw.githubusercontent.com/opencloud-eu/desktop/$CI_COMMIT_SHA/.woodpecker.env - curl -o script.sh https://raw.githubusercontent.com/opencloud-eu/desktop/$CI_COMMIT_SHA/test/gui/woodpecker/script.sh - . ./.woodpecker.env - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc ls --recursive s3/$CACHE_BUCKET/opencloud-build - bash script.sh check_opencloud_cache - environment: - <<: *minio_environment - image: *minio_image - name: check-for-existing-cache - - commands: + + - name: clone-opencloud + image: docker.io/golang:1.24 + commands: - . ./.woodpecker.env - if $OPENCLOUD_CACHE_FOUND; then exit 0; fi - git clone -b $OPENCLOUD_BRANCH --single-branch https://github.com/opencloud-eu/opencloud.git repo_opencloud - cd repo_opencloud - git checkout $OPENCLOUD_COMMITID - image: docker.io/golang:1.24 - name: clone-opencloud - - commands: + + - name: generate-opencloud + image: owncloudci/nodejs:20 + commands: - . ./.woodpecker.env - if $OPENCLOUD_CACHE_FOUND; then exit 0; fi - cd repo_opencloud - for i in $(seq 3); do make node-generate-prod && break || sleep 1; done - image: owncloudci/nodejs:20 - name: generate-opencloud - - commands: + + - name: build-opencloud + image: docker.io/golang:1.24 + commands: - . ./.woodpecker.env - if $OPENCLOUD_CACHE_FOUND; then exit 0; fi - cd repo_opencloud - for i in $(seq 3); do make -C opencloud build && break || sleep 1; done - image: docker.io/golang:1.24 - name: build-opencloud - - commands: + + - name: upload-opencloud-cache + image: *minio_image + environment: + <<: *minio_environment + commands: - . ./.woodpecker.env - if $OPENCLOUD_CACHE_FOUND; then exit 0; fi - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -a repo_opencloud/opencloud/bin/opencloud s3/$CACHE_BUCKET/opencloud-build/$OPENCLOUD_COMMITID/ - mc ls --recursive s3/$CACHE_BUCKET/opencloud-build - environment: - <<: *minio_environment - image: *minio_image - name: upload-opencloud-cache -when: - - branch: - - main - - stable-* - event: - - push - - manual - - event: pull_request - evaluate: | - !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") - - event: tag - - event: cron - cron: nightly* diff --git a/.woodpecker/cache-pnpm.yaml b/.woodpecker/cache-pnpm.yaml index 48a665c816..b8747a455f 100644 --- a/.woodpecker/cache-pnpm.yaml +++ b/.woodpecker/cache-pnpm.yaml @@ -1,4 +1,3 @@ ---- variables: - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' - &minio_environment @@ -8,61 +7,66 @@ variables: from_secret: cache_s3_secret_key CACHE_BUCKET: from_secret: cache_s3_bucket - MC_HOST: "https://s3.ci.opencloud.eu" + MC_HOST: 'https://s3.ci.opencloud.eu' + +when: + - branch: + - main + - stable-* + event: + - push + - manual + - event: tag + - event: pull_request + evaluate: | + !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") + - event: cron + cron: nightly* + +workspace: + base: /woodpecker/ + path: desktop steps: - - commands: + - name: check-browsers-cache + image: *minio_image + environment: + <<: *minio_environment + commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc ls --recursive s3/$CACHE_BUCKET/web - bash test/gui/woodpecker/script.sh check_browsers_cache - environment: - <<: *minio_environment - image: *minio_image - name: check-browsers-cache - - commands: + + - name: pnpm-install + image: owncloudci/nodejs:20 + commands: - . ./.woodpecker.env - if $BROWSER_CACHE_FOUND; then exit 0; fi - cd test/gui/ - npm i -s -g -f "$(jq -r ".packageManager" < webUI/package.json)" - pnpm config set store-dir ./.pnpm-store - make pnpm-install + + - name: install-browsers image: owncloudci/nodejs:20 - name: pnpm-install - - commands: + environment: + PLAYWRIGHT_BROWSERS_PATH: .playwright + commands: - . ./.woodpecker.env - if $BROWSER_CACHE_FOUND; then exit 0; fi - cd test/gui/ - make pnpm-install-chromium - cd webUI - tar -czvf /woodpecker/desktop/playwright-browsers.tar.gz .playwright + + - name: upload-browsers-cache + image: *minio_image environment: - PLAYWRIGHT_BROWSERS_PATH: .playwright - image: owncloudci/nodejs:20 - name: install-browsers - - commands: + <<: *minio_environment + commands: - . ./.woodpecker.env - if $BROWSER_CACHE_FOUND; then exit 0; fi - playwright_version=$(bash test/gui/woodpecker/script.sh get_playwright_version) - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -r -a /woodpecker/desktop/playwright-browsers.tar.gz s3/$CACHE_BUCKET/web/browsers-cache/$playwright_version/ - mc ls --recursive s3/$CACHE_BUCKET/web - environment: - <<: *minio_environment - image: *minio_image - name: upload-browsers-cache -when: - - branch: - - main - - stable-* - event: - - push - - manual - - event: tag - - event: pull_request - evaluate: | - !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") - - event: cron - cron: nightly* -workspace: - base: /woodpecker/ - path: desktop diff --git a/.woodpecker/cache-python.yaml b/.woodpecker/cache-python.yaml index 9056b3b180..0069edd430 100644 --- a/.woodpecker/cache-python.yaml +++ b/.woodpecker/cache-python.yaml @@ -1,4 +1,3 @@ ---- variables: - &squish_image 'opencloudeu/squish@sha256:6eaecc218044020f49f24fd29b6bdc052e8170699a762687b10398b353e5fcda' - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' @@ -9,18 +8,40 @@ variables: from_secret: cache_s3_secret_key CACHE_BUCKET: from_secret: cache_s3_bucket - MC_HOST: "https://s3.ci.opencloud.eu" + MC_HOST: 'https://s3.ci.opencloud.eu' + +when: + - branch: + - main + - stable-* + event: + - push + - manual + - event: tag + - event: pull_request + evaluate: | + !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") + - event: cron + cron: nightly* + +workspace: + base: /woodpecker/ + path: desktop steps: - name: check-python-cache + image: *minio_image + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc ls s3/$CACHE_BUCKET/desktop-build - bash test/gui/woodpecker/script.sh check_python_cache - environment: - <<: *minio_environment - image: *minio_image + - name: install-python-modules + image: *squish_image + environment: + PYTHONUSERBASE: /woodpecker/desktop commands: - . ./.woodpecker.env - if $PYTHON_CACHE_FOUND; then exit 0; fi @@ -28,33 +49,14 @@ steps: - python3.10 -m pip list -v - requirements_sha=$(sha1sum test/gui/requirements.txt | cut -d" " -f1) - tar -czvf /woodpecker/desktop/python-cache-$requirements_sha.tar.gz lib/python3.10/site-packages - image: *squish_image - environment: - PYTHONUSERBASE: /woodpecker/desktop + - name: upload-python-cache + image: *minio_image + environment: + <<: *minio_environment commands: - . ./.woodpecker.env - if $PYTHON_CACHE_FOUND; then exit 0; fi - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -r -a /woodpecker/desktop/python-cache*.tar.gz s3/$CACHE_BUCKET/desktop-build/ - mc ls s3/$CACHE_BUCKET/desktop-build - environment: - <<: *minio_environment - image: *minio_image - -when: - - branch: - - main - - stable-* - event: - - push - - manual - - event: tag - - event: pull_request - evaluate: | - !(CI_COMMIT_SOURCE_BRANCH matches "next-release/(main|stable-*)" && CI_COMMIT_AUTHOR == "openclouders") - - event: cron - cron: nightly* -workspace: - base: /woodpecker/ - path: desktop diff --git a/.woodpecker/notification.yaml b/.woodpecker/notification.yaml index ab61fda32e..53b4440e75 100644 --- a/.woodpecker/notification.yaml +++ b/.woodpecker/notification.yaml @@ -6,14 +6,21 @@ variables: - &ci_woodpecker_url from_secret: oc_ci_url -depends_on: [build, ui-tests] -runs_on: [ success, failure ] when: - - event: [ push , manual ] + - event: [push, manual] branch: ${CI_REPO_DEFAULT_BRANCH} - event: pull_request - event: cron cron: nightly* + +depends_on: + - build + - ui-tests + +runs_on: + - success + - failure + skip_clone: true steps: @@ -36,7 +43,6 @@ steps: CI_WOODPECKER_URL: *ci_woodpecker_url CI_REPO_ID: *current_repo_id CI_WOODPECKER_TOKEN: no-auth-needed-on-this-repo - commands: - git clone --single-branch --branch $QA_REPO_BRANCH $QA_REPO /tmp/qa - cd /tmp/qa/scripts/matrix-notification/ diff --git a/.woodpecker/purge-cache.yaml b/.woodpecker/purge-cache.yaml index a93dd07cda..5a1ffb0688 100644 --- a/.woodpecker/purge-cache.yaml +++ b/.woodpecker/purge-cache.yaml @@ -1,12 +1,11 @@ ---- variables: - - &minio_image "minio/mc:RELEASE.2021-10-07T04-19-58Z" + - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' - &minio_environment AWS_ACCESS_KEY_ID: from_secret: cache_s3_access_key AWS_SECRET_ACCESS_KEY: from_secret: cache_s3_secret_key - MC_HOST: "https://s3.ci.opencloud.eu" + MC_HOST: 'https://s3.ci.opencloud.eu' CACHE_BUCKET: from_secret: cache_s3_bucket PUBLIC_BUCKET: public @@ -41,11 +40,11 @@ matrix: steps: - name: ${JOB_NAME} image: *minio_image + environment: + <<: *minio_environment commands: - mc alias set s3 "$MC_HOST" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" - if [ "$USE_PUBLIC_BUCKET" = "true" ]; then CACHE_BUCKET=$PUBLIC_BUCKET; fi - to_delete=$(mc find "s3/$CACHE_BUCKET/${PURGE_PATH}" --older-than "${TTL}") - if [ "$to_delete" = "" ]; then exit 0; fi - mc rm $to_delete - environment: - <<: *minio_environment diff --git a/.woodpecker/ready-release-go.yaml b/.woodpecker/ready-release-go.yaml index bd98b7972f..f4f15c1996 100644 --- a/.woodpecker/ready-release-go.yaml +++ b/.woodpecker/ready-release-go.yaml @@ -3,7 +3,7 @@ when: branch: ${CI_REPO_DEFAULT_BRANCH} steps: - release-helper: + - name: release-helper image: woodpeckerci/plugin-ready-release-go:latest settings: git_email: devops@opencloud.eu diff --git a/.woodpecker/translation.yaml b/.woodpecker/translation.yaml index e0e276a4e9..6b44414195 100644 --- a/.woodpecker/translation.yaml +++ b/.woodpecker/translation.yaml @@ -1,15 +1,17 @@ ---- -when: - - event: cron - cron: translation-sync - variables: - &ubuntu_image 'ubuntu' - &git_action_plugin 'quay.io/thegeeklab/wp-git-action:2' +when: + - event: cron + cron: translation-sync + steps: - name: translation-update image: *ubuntu_image + environment: + TX_TOKEN: + from_secret: tx_token commands: - apt update - ln -fs /usr/share/zoneinfo/Europe/Berlin /etc/localtime @@ -23,9 +25,6 @@ steps: - rm -rf /tmp/tx - tx pull --force --all - rm tx LICENSE - environment: - TX_TOKEN: - from_secret: tx_token - name: translation-push image: *git_action_plugin diff --git a/.woodpecker/ui-tests.yaml b/.woodpecker/ui-tests.yaml index f45ad7a1d5..52d1d49943 100644 --- a/.woodpecker/ui-tests.yaml +++ b/.woodpecker/ui-tests.yaml @@ -1,4 +1,3 @@ ---- variables: - &squish_image 'opencloudeu/squish@sha256:6eaecc218044020f49f24fd29b6bdc052e8170699a762687b10398b353e5fcda' - &minio_image 'minio/mc:RELEASE.2021-10-07T04-19-58Z' @@ -13,59 +12,79 @@ variables: MC_HOST: 'https://s3.ci.opencloud.eu' PUBLIC_BUCKET: public +when: + - branch: + - main + - stable-* + event: + - push + - manual + - event: pull_request + - event: tag + - event: cron + cron: nightly* + depends_on: - cache-opencloud - cache-pnpm - build + +workspace: + base: /woodpecker + path: desktop + steps: - - commands: + - name: pnpm-install + image: owncloudci/nodejs:20 + commands: - cd test/gui/ - npm i -s -g -f "$(jq -r ".packageManager" < webUI/package.json)" - pnpm config set store-dir ./.pnpm-store - make pnpm-install - image: owncloudci/nodejs:20 - name: pnpm-install + - name: restore-python-cache + image: *minio_image + environment: + <<: *minio_environment commands: - requirements_sha=$(sha1sum test/gui/requirements.txt | cut -d" " -f1) - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -a s3/$CACHE_BUCKET/desktop-build/python-cache-$requirements_sha.tar.gz /woodpecker/desktop + + - name: install-python-modules + image: *squish_image environment: - <<: *minio_environment - image: *minio_image - - commands: + PYTHONUSERBASE: /woodpecker/desktop/ + commands: - tar -xvf python-cache-*.tar.gz -C . - make -C test/gui/ pip-install - python3.10 -m pip list -v - image: *squish_image + + - name: restore-browsers-cache + image: *minio_image environment: - PYTHONUSERBASE: /woodpecker/desktop/ - name: install-python-modules - - commands: + <<: *minio_environment + commands: - playwright_version=$(bash test/gui/woodpecker/script.sh get_playwright_version) - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -r -a s3/$CACHE_BUCKET/web/browsers-cache/$playwright_version/playwright-browsers.tar.gz /woodpecker/desktop + + - name: unzip-browsers-cache + image: owncloud/ubuntu:20.04 + commands: + - tar -xvf /woodpecker/desktop/playwright-browsers.tar.gz -C . + + - name: restore-opencloud-cache + image: *minio_image environment: <<: *minio_environment - image: *minio_image - name: restore-browsers-cache - - commands: - - tar -xvf /woodpecker/desktop/playwright-browsers.tar.gz -C . - image: owncloud/ubuntu:20.04 - name: unzip-browsers-cache - - commands: + commands: - . ./.woodpecker.env - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -r -a s3/$CACHE_BUCKET/opencloud-build/$OPENCLOUD_COMMITID/opencloud /woodpecker/desktop - environment: - <<: *minio_environment - image: *minio_image - name: restore-opencloud-cache - - commands: - - mkdir -p /srv/app/tmp/opencloud/opencloud/data/ - - mkdir -p /srv/app/tmp/opencloud/storage/users/ - - ./opencloud init - - ./opencloud server + + - name: opencloud + image: docker.io/golang:1.24 detach: true environment: FRONTEND_SEARCH_MIN_LENGTH: '2' @@ -80,20 +99,25 @@ steps: OC_URL: https://opencloud:9200 PROXY_ENABLE_BASIC_AUTH: true WEB_UI_CONFIG_FILE: /woodpecker/desktop/test/gui/woodpecker/config-opencloud.json - image: docker.io/golang:1.24 - name: opencloud - - commands: - - timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin https://opencloud:9200/graph/v1.0/users/admin -w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done' + commands: + - mkdir -p /srv/app/tmp/opencloud/opencloud/data/ + - mkdir -p /srv/app/tmp/opencloud/storage/users/ + - ./opencloud init + - ./opencloud server + + - name: wait-for-opencloud image: owncloudci/alpine:latest - name: wait-for-opencloud + commands: + - timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin https://opencloud:9200/graph/v1.0/users/admin -w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done' + - name: restore-desktop-client + image: *minio_image + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -a -r s3/$CACHE_BUCKET/desktop-build/${CI_COMMIT_SHA}/ /woodpecker/desktop/build - ls -lh /woodpecker/desktop/build/bin - environment: - <<: *minio_environment - image: *minio_image - name: create-extra-directories image: *ubuntu_image @@ -131,55 +155,42 @@ steps: - name: crash-log image: *ubuntu_image when: - - status: [success, failure] + - status: + - success + - failure commands: - cat /woodpecker/desktop/test/gui/tmp/OpenCloud-crash.log 2>/dev/null || exit 0 - name: upload-test-reports - environment: - <<: *minio_environment image: *minio_image when: - - status: [failure] + - status: failure + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - mc cp -a -r /woodpecker/desktop/test/gui/clientLog/* s3/$PUBLIC_BUCKET/desktop/$CI_PIPELINE_NUMBER/logs/ - mc cp -a -r /woodpecker/desktop/test/gui/guiReportUpload s3/$PUBLIC_BUCKET/desktop/$CI_PIPELINE_NUMBER - name: gui-test-reports - environment: - <<: *minio_environment image: *minio_image when: - - status: [failure] + - status: failure + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - bash test/gui/woodpecker/gui_test_reports.sh - name: client-log - environment: - <<: *minio_environment image: *minio_image when: - - status: [failure] + - status: failure + environment: + <<: *minio_environment commands: - mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY - cd /woodpecker/desktop/test/gui/clientLog/ - echo "To download the logs, access the following links:" - logs=$(mc find s3/$PUBLIC_BUCKET/desktop/$CI_PIPELINE_NUMBER/logs/) - "for f in $logs; do echo \"$MC_HOST/$f \n \" | cut -d '/' -f1-3,5-99; done" - -when: - - branch: - - main - - stable-* - event: - - push - - manual - - event: pull_request - - event: tag - - event: cron - cron: nightly* -workspace: - base: /woodpecker - path: desktop