cmake-test.yml

name: Test-Build
on:
  push:
    branches:
      - main
      - develop
  pull_request:
    branches:
      - main
      - develop

jobs:
  test-build:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    defaults:
      run:
        shell: bash

    strategy:
      matrix:
        build_type: [Release]
        memory_test: [ON, OFF]
        compiler: [g++]
        language: ['cpp']
        python_api: [OFF, ON]
        exclude:
          - build_type: Release
            memory_test: ON
          - build_type: RelWithDebInfo
            memory_test: OFF
          - memory_test: ON
            python_api: ON
      
    steps:

    - name: Update packages
      run: sudo apt-get update

    - name: Install openmpi
      run: sudo apt-get install -yq libopenmpi-dev openmpi-bin

    - name: Install cmake
      run: sudo apt-get install -yq cmake

    - name: Install Python and dependencies
      if: matrix.python_api == 'ON'
      run: |
        sudo apt-get install -yq python3 python3-pip python3-dev python3-pybind11 pybind11-dev
        pip3 install numpy

    - uses: actions/checkout@v4

    - name: build Catch2
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'Catch2'
        repo-path: 'catchorg/Catch2'
        repo-ref: 'v3.11.0'
        cache: true

    - name: build kokkos
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'kokkos'
        repo-path: 'kokkos/kokkos'
        repo-ref: '4.6.01'
        cache: true
        cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
        options: '-DCMAKE_CXX_STANDARD=17
                  -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                  -DKokkos_ENABLE_SERIAL=ON
                  -DKokkos_ENABLE_OPENMP=OFF
                  -DKokkos_ENABLE_CUDA=OFF
                  -DKokkos_ENABLE_CUDA_LAMBDA=OFF
                  -DKokkos_ENABLE_CUDA_CONSTEXPR=OFF'

    - name: build kokkos-kernels
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'kokkos-kernels'
        repo-path: 'kokkos/kokkos-kernels'
        repo-ref: '4.6.01'
        cache: true
        cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
        options: '-DCMAKE_CXX_STANDARD=17
                  -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                  -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'

    - name: build omega_h
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'omega_h'
        repo-path: 'SCOREC/omega_h'
        repo-ref: '1765836a00b9a64b8b3791f1442ac52f147e43b2'
        cache: true
        cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
        options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                  -DCMAKE_C_COMPILER=`which mpicc`
                  -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                  -DOmega_h_USE_MPI=ON
                  -DOmega_h_USE_Kokkos=ON
                  -DBUILD_TESTING=OFF
                  -DMPIEXEC_EXECUTABLE=`which mpirun`
                  -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'
    
    - name: build meshFields
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'meshFields'
        repo-path: 'SCOREC/meshFields'
        repo-ref: 'b1482bbba288df210784b2345eae08e34faabdc4'
        cache: true
        cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
        options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                  -DCMAKE_C_COMPILER=`which mpicc`
                  -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                  -DMPIEXEC_EXECUTABLE=`which mpirun`
                  -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos
                  -DOmega_h_DIR=${{ runner.temp }}/build-omega_h/install/lib/cmake/Omega_h'

    - name: build perfstubs
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'perfstubs'
        repo-path: 'UO-OACISS/perfstubs'
        repo-ref: 'a1fa3feb1d89214e28047f166500351074b5f0c2'
        cache: true
        options: '-DCMAKE_CXX_COMPILER=mpicxx'

    - name: build ADIOS2
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'ADIOS2'
        repo-path: 'ornladios/ADIOS2'
        repo-ref: 'v2.10.2'
        cache: true
        options: '-DADIOS2_USE_CUDA=OFF'

    - name: build redev
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'redev'
        repo-path: 'SCOREC/redev'
        repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
        cache: true
        cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
        options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                  -DMPIEXEC_EXECUTABLE=`which mpirun`
                  -DCMAKE_BUILD_TYPE=Release
                  -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                  -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
                  -Dperfstubs_DIR=${{ runner.temp }}/build-perfstubs/install/lib/cmake'

    - name: clone petsc
      id: clone-petsc
      run: |
        cd ${{ runner.temp }}
        git clone -b v3.24.2 https://gitlab.com/petsc/petsc.git petsc
        cd petsc
        echo "petsc-commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

    - name: Cache PETSc
      id: cache-petsc
      uses: actions/cache@v3
      with:
        path: ${{ runner.temp }}/petsc
        key: build-petsc-${{ steps.clone-petsc.outputs.petsc-commit-hash }}
    
    - name: build petsc
      if: steps.cache-petsc.outputs.cache-hit != 'true'
      run: |
        cd ${{ runner.temp }}/petsc
        ./configure \
          PETSC_ARCH=ubuntu-kokkos \
          --with-kokkos-dir="${{ runner.temp }}/build-kokkos/install/" \
          --with-kokkos-kernels-dir="${{ runner.temp }}/build-kokkos-kernels/install/" \
          --with-cuda=0 \
          --with-shared-libraries=0 \
          --download-fblaslapack
        make all check

    - name: checkout pcms_testcases
      uses: actions/checkout@v3
      with:
        repository: jacobmerson/pcms_testcases
        path: pcms_testcases

    - name: Install fftw3
      run: sudo apt-get install -yq libfftw3-dev pkg-config

    - name: Set LD_LIBRARY_PATH for shared libraries
      if: matrix.python_api == 'ON'
      run: |
        echo "LD_LIBRARY_PATH=${{ runner.temp }}/build-kokkos/install/lib:${{ runner.temp }}/build-omega_h/install/lib:${{ runner.temp }}/build-meshFields/install/lib:${{ runner.temp }}/build-redev/install/lib:${{ runner.temp }}/build-ADIOS2/install/lib:${{ runner.temp }}/build-perfstubs/install/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV

    - name: build pcms
      uses: ./.github/actions/install-repo
      with:
        repo-name: 'pcms'
        repo-path: 'SCOREC/pcms'
        repo-ref: ''
        cache: false
        options: '-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
                  -DCMAKE_C_COMPILER=`which mpicc`
                  -DCMAKE_CXX_COMPILER=`which mpicxx`
                  -DCMAKE_Fortran_COMPILER=`which mpifort`
                  -DMPIEXEC_PREFLAGS="--oversubscribe"
                  -DPCMS_ENABLE_PETSC=ON
                  -DPETSC_LINK_STATIC=ON
                  -DPETSC_DIR=${{ runner.temp }}/petsc
                  -DPETSC_ARCH=ubuntu-kokkos
                  -DPCMS_TIMEOUT=10
                  -DPCMS_ENABLE_Python=${{ matrix.python_api }}
                  -DCatch2_DIR=${{ runner.temp }}/build-Catch2/install/lib/cmake/Catch2
                  -DOmega_h_DIR=${{ runner.temp }}/build-omega_h/install/lib/cmake/Omega_h
                  -Dmeshfields_DIR=${{ runner.temp }}/build-meshFields/install/lib/cmake/meshfields
                  -Dredev_DIR=${{ runner.temp }}/build-redev/install/lib/cmake/redev
                  -DMPIEXEC_EXECUTABLE=`which mpirun`
                  -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
                  -Dperfstubs_DIR=${{ runner.temp }}/build-perfstubs/install/lib/cmake
                  -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos
                  -DKokkosKernels_DIR=${{ runner.temp }}/build-kokkos-kernels/install/lib/cmake/KokkosKernels/
                  -DPCMS_TEST_DATA_DIR=$PWD/pcms_testcases'

    # - name: Install Valgrind
    #   run: sudo apt-get install -yq valgrind

    - name: Run CTest
      run: ctest --test-dir ${{ runner.temp }}/build-pcms --output-on-failure

    - name: Print Test
      if: always()
      run: cat ${{ runner.temp }}/build-pcms/Testing/Temporary/LastTest.log

    - name: Run Python API tests
      if: matrix.python_api == 'ON'
      run: |
        export PYTHONPATH=${{ runner.temp }}/build-pcms/install/lib/python3.12/site-packages:$PYTHONPATH
        cd ${{ github.workspace }}/src/pcms/pythonapi
        python3 test_file_io.py
        python3 test_omega_h_field.py
        python3 test_uniform_grid_field.py
        python3 test_field_copy.py
        python3 test_mls_interpolation.py

    - name: Test PCMS Installation
      if: matrix.python_api == 'OFF'
      shell: bash
      run: |
        # Configure and build the test
        # it should only ask for pcms_DIR/ROOT to find all dependencies
        ls ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms
        echo "-------- PCMS CONFIG -------------"
        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-config.cmake
        echo "-------- PCMS TARGETS -------------"
        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-targets.cmake
        echo "-------- PCMS TRANSFER TARGETS -------------"
        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms_transfer-targets.cmake
        export VERBOSE=1
        cmake \
          -B ${{github.workspace}}/examples/external-usage-example/build \
          -S ${{github.workspace}}/examples/external-usage-example/ \
          -Dpcms_DIR=${{ runner.temp }}/build-pcms/install/lib/cmake/pcms \
          -DPETSC_LINK_STATIC=ON \
          -DPETSC_DIR=${{ runner.temp }}/petsc \
          -DPETSC_ARCH=ubuntu-kokkos \
          --debug-output
        cmake --build ${{github.workspace}}/examples/external-usage-example/build