project-template/.github/workflows/ci.yml at main · dataciviclab/project-template · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
name: CI

on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:
    inputs:
      run_smoke:
        description: "Run optional smoke e2e job"
        required: false
        type: boolean
        default: false

env:
  PYTHON_VERSION: "3.11"
  TOOLKIT_PIP_PACKAGE: ""
  TOOLKIT_BIN: "toolkit"

jobs:
  contract:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install contract test dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install pytest pyyaml

      - name: Verify required files
        run: |
          test -f dataset.yml
          test -f sql/clean.sql
          test -f scripts/smoke.sh

      - name: Run contract tests
        run: pytest tests/test_contract.py

  smoke:
    if: ${{ vars.RUN_SMOKE == '1' || (github.event_name == 'workflow_dispatch' && inputs.run_smoke) }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install smoke dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install pytest pyyaml
          if [ -n "${TOOLKIT_PIP_PACKAGE}" ]; then
            python -m pip install "${TOOLKIT_PIP_PACKAGE}"
          else
            git clone --depth 1 https://github.com/dataciviclab/toolkit.git .toolkit-src
            python -m pip install -e ./.toolkit-src
          fi

      - name: Export DCL_ROOT
        run: echo "DCL_ROOT=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}"

      - name: Smoke run all validate all status
        run: sh scripts/smoke.sh

      - name: Upload minimal artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: toolkit-artifacts
          path: |
            _smoke_out/data/_runs/**/*.json
            _smoke_out/data/raw/**/*.json
            _smoke_out/data/clean/**/*.json
            _smoke_out/data/mart/**/*.json
          if-no-files-found: warn