diff --git a/.github/workflows/prepare_test_data.yml b/.github/workflows/prepare_test_data.yml new file mode 100644 index 000000000..bf7f85407 --- /dev/null +++ b/.github/workflows/prepare_test_data.yml @@ -0,0 +1,77 @@ +name: Prepare test data release + +on: + workflow_dispatch: + inputs: + zenodo_record_id: + description: "Zenodo record ID to download the dataset from" + type: string + workflow_call: + inputs: + zenodo_record_id: + description: "Zenodo record ID to download the dataset from" + type: string + # pull_request: + # types: [opened, synchronize, reopened, ready_for_review] + # paths: + # - .github/workflows/prepare_test_data.yml + +env: + RELEASE_TAG_TEMPLATE: testdata_zenodo. + ARCHIVE_NAME: test_data.tar.zst + +jobs: + build-and-release: + name: Download and publish test data + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + - name: Check if release exists + id: checkrelease + run: | + if gh release view ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} >/dev/null 2>&1; then + echo "release_exists=true" >> $GITHUB_OUTPUT + else + echo "release_exists=false" >> $GITHUB_OUTPUT + fi + - name: Release already exists, skipping + if: steps.checkrelease.outputs.release_exists == 'true' + run: echo "Release ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} already exists. Skipping workflow." + + - uses: actions/setup-python@v5 + if: steps.checkrelease.outputs.release_exists != 'true' + with: + python-version: "3.10" + + - name: Install tools + if: steps.checkrelease.outputs.release_exists != 'true' + run: | + python -m pip install --upgrade pip + python -m pip install zenodo-get + + - name: Download dataset from Zenodo + if: steps.checkrelease.outputs.release_exists != 'true' + run: | + mkdir -p test_data + cd test_data + zenodo_get ${{ inputs.zenodo_record_id }} + cd - + + - name: Create compressed archive + if: steps.checkrelease.outputs.release_exists != 'true' + run: | + tar -I zstd -cf "${ARCHIVE_NAME}" test_data/ + + - name: Publish Release asset (creates or updates a prerelease) + if: steps.checkrelease.outputs.release_exists != 'true' + run: | + gh release create \ + ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} \ + --prerelease \ + --title ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} \ + --notes "Automated test data release for Zenodo record ID ${{ inputs.zenodo_record_id }}" \ + ${{ env.ARCHIVE_NAME }} + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT }} diff --git a/.github/workflows/release_pypi.yml b/.github/workflows/release_pypi.yml index e378911d8..6501aa9b7 100644 --- a/.github/workflows/release_pypi.yml +++ b/.github/workflows/release_pypi.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: release: types: - - published + - released jobs: build: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 62bfa4950..48a04a4c7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,6 +1,7 @@ name: Build and Test env: ZENODO_RECORD_ID: 17423608 + on: push: branches: @@ -15,8 +16,26 @@ on: branches: - main - develop + jobs: + constants: + name: Set constants + runs-on: ubuntu-latest + outputs: + ZENODO_RECORD_ID: ${{ steps.output_env.outputs.ZENODO_RECORD_ID }} + steps: + - id: output_env + run: echo "ZENODO_RECORD_ID=${{ env.ZENODO_RECORD_ID }}" >> $GITHUB_OUTPUT + prepare_test_data: + name: Prepare test data + needs: constants + uses: ./.github/workflows/prepare_test_data.yml + secrets: inherit + with: + zenodo_record_id: ${{ needs.constants.outputs.ZENODO_RECORD_ID }} + test_and_build: + needs: prepare_test_data if: github.event.pull_request.draft == false runs-on: ${{ matrix.os }} strategy: @@ -32,31 +51,14 @@ jobs: dependencies: testing python-version: ${{ matrix.python-version }} token: ${{ secrets.GITHUB_TOKEN }} - - name: Set data dir - shell: bash - id: set-path - run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT" - - name: Restore dataset cache - id: cache - uses: actions/cache@v4 - with: - path: ${{ steps.set-path.outputs.data_dir }} - key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }} - - name: Install zenodo-get - if: steps.cache.outputs.cache-hit != 'true' - run: python3 -m pip install zenodo-get - shell: bash - - name: Download test dataset - shell: bash - if: steps.cache.outputs['cache-hit'] != 'true' + - name: Download test data archive from release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release download testdata_zenodo.${{ env.ZENODO_RECORD_ID }} --pattern "test_data.tar.zst" + - name: Extract test data run: | - mkdir -p ${{ steps.set-path.outputs.data_dir }} - cd ${{ steps.set-path.outputs.data_dir }} - zenodo_get $ZENODO_RECORD_ID - cd - - - name: Uninstall zenodo-get - if: steps.cache.outputs['cache-hit'] != 'true' - run: python3 -m pip uninstall --yes zenodo-get - shell: bash + tar -x -f test_data.tar.zst + rm test_data.tar.zst - name: Run pytest run: pytest -v