Create test data asset

.github/workflows/prepare_test_data.yml

@@ -0,0 +1,77 @@
+name: Prepare test data release
+
+on:
+    workflow_dispatch:
+        inputs:
+            zenodo_record_id:
+                description: "Zenodo record ID to download the dataset from"
+                type: string
+    workflow_call:
+        inputs:
+            zenodo_record_id:
+                description: "Zenodo record ID to download the dataset from"
+                type: string
+    # pull_request:
+    #     types: [opened, synchronize, reopened, ready_for_review]
+    #     paths:
+    #         - .github/workflows/prepare_test_data.yml
+
+env:
+    RELEASE_TAG_TEMPLATE: testdata_zenodo.
+    ARCHIVE_NAME: test_data.tar.zst
+
+jobs:
+    build-and-release:
+        name: Download and publish test data
+        runs-on: ubuntu-latest
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        steps:
+            - uses: actions/checkout@v4
+            - name: Check if release exists
+              id: checkrelease
+              run: |
+                  if gh release view ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} >/dev/null 2>&1; then
+                      echo "release_exists=true" >> $GITHUB_OUTPUT
+                  else
+                      echo "release_exists=false" >> $GITHUB_OUTPUT
+                  fi
+            - name: Release already exists, skipping
+              if: steps.checkrelease.outputs.release_exists == 'true'
+              run: echo "Release ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} already exists. Skipping workflow."
+
+            - uses: actions/setup-python@v5
+              if: steps.checkrelease.outputs.release_exists != 'true'
+              with:
+                  python-version: "3.10"
+
+            - name: Install tools
+              if: steps.checkrelease.outputs.release_exists != 'true'
+              run: |
+                  python -m pip install --upgrade pip
+                  python -m pip install zenodo-get
+
+            - name: Download dataset from Zenodo
+              if: steps.checkrelease.outputs.release_exists != 'true'
+              run: |
+                  mkdir -p test_data
+                  cd test_data
+                  zenodo_get ${{ inputs.zenodo_record_id }}
+                  cd -
+
+            - name: Create compressed archive
+              if: steps.checkrelease.outputs.release_exists != 'true'
+              run: |
+                  tar -I zstd -cf "${ARCHIVE_NAME}" test_data/
+
+            - name: Publish Release asset (creates or updates a prerelease)
+              if: steps.checkrelease.outputs.release_exists != 'true'
+              run: |
+                  gh release create \
+                      ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} \
+                      --prerelease \
+                      --title ${{ env.RELEASE_TAG_TEMPLATE }}${{ inputs.zenodo_record_id }} \
+                      --notes "Automated test data release for Zenodo record ID ${{ inputs.zenodo_record_id }}" \
+                      ${{ env.ARCHIVE_NAME }}
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GH_PAT }}

.github/workflows/release_pypi.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   release:
     types:
-      - published
+      - released
 
 jobs:
   build:

.github/workflows/testing.yml

@@ -1,6 +1,7 @@
 name: Build and Test
 env:
   ZENODO_RECORD_ID: 17423608
+
 on:
   push:
     branches:
@@ -15,8 +16,26 @@ on:
     branches:
       - main
       - develop
+
 jobs:
+  constants:
+    name: Set constants
+    runs-on: ubuntu-latest
+    outputs:
+      ZENODO_RECORD_ID: ${{ steps.output_env.outputs.ZENODO_RECORD_ID }}
+    steps:
+      - id: output_env
+        run: echo "ZENODO_RECORD_ID=${{ env.ZENODO_RECORD_ID }}" >> $GITHUB_OUTPUT
+  prepare_test_data:
+    name: Prepare test data
+    needs: constants
+    uses: ./.github/workflows/prepare_test_data.yml
+    secrets: inherit
+    with:
+      zenodo_record_id: ${{ needs.constants.outputs.ZENODO_RECORD_ID }}
+
   test_and_build:
+    needs: prepare_test_data
     if: github.event.pull_request.draft == false
     runs-on: ${{ matrix.os }}
     strategy:
@@ -32,31 +51,14 @@ jobs:
           dependencies: testing
           python-version: ${{ matrix.python-version }}
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set data dir
-        shell: bash
-        id: set-path
-        run: echo "data_dir=$GITHUB_WORKSPACE/test_data" >> "$GITHUB_OUTPUT"
-      - name: Restore dataset cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.set-path.outputs.data_dir }}
-          key: eitprocessing-testdata-zenodo.${{ env.ZENODO_RECORD_ID }}
-      - name: Install zenodo-get
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: python3 -m pip install zenodo-get
-        shell: bash
-      - name: Download test dataset
-        shell: bash
-        if: steps.cache.outputs['cache-hit'] != 'true'
+      - name: Download test data archive from release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release download testdata_zenodo.${{ env.ZENODO_RECORD_ID }} --pattern "test_data.tar.zst"
+      - name: Extract test data
         run: |
-          mkdir -p ${{ steps.set-path.outputs.data_dir }}
-          cd ${{ steps.set-path.outputs.data_dir }}
-          zenodo_get $ZENODO_RECORD_ID
-          cd -
-      - name: Uninstall zenodo-get
-        if: steps.cache.outputs['cache-hit'] != 'true'
-        run: python3 -m pip uninstall --yes zenodo-get
-        shell: bash
+          tar -x -f test_data.tar.zst
+          rm test_data.tar.zst
       - name: Run pytest
         run: pytest -v