.gitlab-ci.yml

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_MESSAGE =~ /CI BUILD/'
    - if: '$CI_COMMIT_MESSAGE =~ /SPACK BUILD/'
    - if: '$FORCE_CI == "true"'
    - if: '$FORCE_SPACK == "true"'
    - when: never

stages:
  - build
  - test
  - spack-test

variables:
  CCACHE_BASEDIR: $CI_PROJECT_DIR
  CCACHE_DIR: "${CI_BUILDS_DIR}/ccache"
  GIT_CLEAN_FLAGS: none

cache:
  - key: ccache-global
    paths:
      - $CCACHE_DIR
    policy: pull-push
  - key: spack-builds
    paths:
      - "${CI_BUILDS_DIR}/spack-cache"
    when: on_success

# Check ccache integration first
check-ccache:
  stage: build
  script:
    - |
      echo "=== Checking ccache integration ==="
      which ccache || echo "ccache not found in PATH"
      ccache --version || echo "ccache not working"
      echo "CMAKE_CXX_COMPILER_LAUNCHER: ${CMAKE_CXX_COMPILER_LAUNCHER:-not set}"
      echo "CMAKE_C_COMPILER_LAUNCHER: ${CMAKE_C_COMPILER_LAUNCHER:-not set}"
      echo "=== Checking Python version ==="
      python3 --version || echo "python3 not found"
      python3.11 --version || echo "python3.11 not found"
      /usr/bin/python3.11 --version || echo "/usr/bin/python3.11 not found"
      echo "=== Checking system info ==="
      cat /etc/redhat-release
      nproc
      df -h ${CI_BUILDS_DIR}
  rules:
    - when: manual
  allow_failure: true

build-job:
  stage: build
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
      changes:
        - "**/*"
        - "!README.md"
        - "!scripts/**/*"
        - "!doc/**/*"
        - "!spack-repo/**/*"
    - if: '$CI_COMMIT_MESSAGE =~ /CI BUILD/'
    - if: '$FORCE_CI == "true"'
    - when: never
  before_script:
    # Ensure submodules are in sync with the index
    - git submodule sync
    - git submodule update --init --recursive
    - export CCACHE_BASEDIR="$PWD"
    - export CCACHE_COMPILERCHECK=content
    - export CCACHE_MAXSIZE=5G
    - mkdir -p $CCACHE_DIR
    - if [ -d "$CCACHE_DIR" ] && [ "$(ls -A $CCACHE_DIR)" ]; then echo "Using existing ccache with $(ccache --show-stats | grep 'files in cache')"; else echo "Starting with empty ccache"; fi
    - ccache --show-stats || true
    - git clean -ffdx
    # Create temporary Python virtual environment for building outside the Libra path
    - export BUILD_VENV_DIR="${CI_BUILDS_DIR}/build_venv_${CI_PIPELINE_ID}"
    - python3.11 --version
    - python3.11 -m venv "${BUILD_VENV_DIR}"
    - source "${BUILD_VENV_DIR}/bin/activate"
    - pip install --upgrade pip setuptools wheel
    - pip install pytest numpy
    - python --version
    - which python
  script:
    - echo "=== Compiling with ccache integration ==="
    - source "${BUILD_VENV_DIR}/bin/activate"
    - echo $PATH
    - python --version
    - nvidia-smi
    - which nvcc
    - mkdir -p build
    - cd build
    - cmake -DApps_BUILD_TESTS=ON -DNCORES=8 -DKokkos_CUDA_ARCH_NAME=Kokkos_ARCH_VOLTA70 ..
    - make
  after_script:
    - echo "=== ccache statistics ==="
    - ccache --show-stats || true
    - ccache --show-config || true

test-job:
  stage: test
  needs:
    - job: build-job
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
      changes:
        - "**/*"
        - "!README.md"
        - "!scripts/**/*"
        - "!doc/**/*"
        - "!spack-repo/**/*"
    - if: '$CI_COMMIT_MESSAGE =~ /CI BUILD/'
    - if: '$FORCE_CI == "true"'
    - when: never
  script: |
    # Ensure we are in the project root directory
    cd "${CI_PROJECT_DIR}"
    echo "Current working directory: $PWD"
    base_path=$PWD

    export LD_LIBRARY_PATH="${base_path}/install/lib:${LD_LIBRARY_PATH:-}"
    export PYTHONPATH="${base_path}/install/lib:${PYTHONPATH:-}"
    export BUILD_VENV_DIR="${CI_BUILDS_DIR}/build_venv_${CI_PIPELINE_ID}"
    source "${BUILD_VENV_DIR}/bin/activate"
    echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
    echo "PYTHONPATH: $PYTHONPATH"

    mkdir -p ${base_path}/libra-python/reports

    # Ensure libra-python is importable for pytest
    export PYTHONPATH="${base_path}/libra-python:${PYTHONPATH:-}"

    # Symlink test data if needed for both python and cpp tests
    cd ${base_path}/libra-python
    if [ -d "${base_path}/apps/src/tests/gold_standard" ] && [ ! -e "gold_standard" ]; then
      ln -sf ../../apps/src/tests/gold_standard gold_standard
      echo "Symlinked test data for tests"
    fi

    # --- C++ GTest section ---
    cd ${base_path}/apps/src/tests/
    # Run C++ gtest binaries if they exist
    #test_binaries=("LibRATests" "test_roadrunner_robustMinus2" "test_roadrunner_robustPlus2" "test_roadrunner_snrpsf" "test_roadrunner_weight")
    test_binaries=("LibRATests")
    
    for test_binary in "${test_binaries[@]}"; do
        if [ -x "${base_path}/install/bin/tests/${test_binary}" ]; then
            echo "Running ${test_binary}..."
            "${base_path}/install/bin/tests/${test_binary}" --gtest_output=xml:"${base_path}/libra-python/reports/cpp_${test_binary}_report.xml"
        else
            echo "C++ test binary not found: ${base_path}/install/bin/tests/${test_binary}"
        fi
    done
    
    # --- Python pytest section ---
    cd "${base_path}/apps/src/tests/"
    pytest ${base_path}/libra-python/tests --junitxml=pytest_report.xml -v || echo "Pytest failed, but continuing for CI artifact collection"
  artifacts:
    when: always
    reports:
      junit:
        - ${CI_PROJECT_DIR}/apps/src/tests/pytest_report.xml
        - ${CI_PROJECT_DIR}/apps/src/tests/cpp_report.xml
    expire_in: 1 week
  after_script: |
    echo "=== Test job completed ==="
    echo "Current working directory: $PWD"
    echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
    echo "PYTHONPATH: $PYTHONPATH"
    echo "Python version: $(python --version)"
    echo "CMake version: $(cmake --version)"
    echo "ccache stats:"
    ccache --show-stats || true
    echo "=== Cleaning up venv ==="
    rm -rf "${CI_BUILDS_DIR}/build_venv_${CI_PIPELINE_ID}"
    rm -rf ${base_path}/libra-python/reports
    rm -rf ${base_path}/cpp_test_dir
  timeout: 1h

spack-test:
  needs: []
  stage: spack-test
  variables:
    SPACK_ROOT: "${CI_BUILDS_DIR}/spack-installs/spack-persistent"
    SPACK_TEST_BASE_ENV: "spack-test-base"
    LIBRA_TEST_ENV: "libra-test-${CI_PIPELINE_ID}"
    SPACK_CACHE_DIR: "${CI_BUILDS_DIR}/spack-cache"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      changes:
        - spack-repo/**/*
    - if: '$CI_COMMIT_BRANCH == "main"'
      changes:
        - spack-repo/**/*
    - when: manual
      allow_failure: true
  before_script:
    # Ensure submodules are in sync with the index
    - git submodule sync
    - git submodule update --init --recursive
    - echo "=== RHEL8 Spack Setup ==="
    - cat /etc/redhat-release

    - mkdir -p ${SPACK_ROOT}
    - mkdir -p ${SPACK_CACHE_DIR}
    - mkdir -p ${CI_BUILDS_DIR}/spack-installs

    # Setup ccache for spack
    - export CCACHE_DIR="${SPACK_CACHE_DIR}/ccache"
    - export CCACHE_MAXSIZE=5G
    - mkdir -p ${CCACHE_DIR}

    # Create or reuse persistent Python virtual environment for spack
    - export SPACK_VENV_DIR="${CI_BUILDS_DIR}/spack_venv_persistent"
    - python3.11 --version
    - |
      if [ ! -d "${SPACK_VENV_DIR}" ]; then
        echo "Creating persistent spack Python venv..."
        python3.11 -m venv "${SPACK_VENV_DIR}"
        source "${SPACK_VENV_DIR}/bin/activate"
        pip install --upgrade pip setuptools wheel
      else
        echo "Using existing persistent spack Python venv..."
        source "${SPACK_VENV_DIR}/bin/activate"
      fi
    - export PYTHON_CMD=$(which python)
    - ${PYTHON_CMD} --version

    # Clone spack if not exists
    - |
      if [ ! -d "${SPACK_ROOT}/.git" ]; then
        echo "Cloning fresh spack..."
          git clone --depth=1 --branch v1.0.0 https://github.com/spack/spack.git ${SPACK_ROOT}
      else
        echo "Using existing spack installation"
        cd ${SPACK_ROOT} && git pull
      fi

    # Setup spack environment
    - source ${SPACK_ROOT}/share/spack/setup-env.sh

    # Use a minimal/vanilla spack config for RHEL8 (store config in ${SPACK_ROOT}/.spack, not ~/.spack)
    - mkdir -p ${SPACK_ROOT}/.spack
    - |
      cat > ${SPACK_ROOT}/.spack/config.yaml << EOF
      config: {}
      EOF
    - export SPACK_USER_CONFIG_PATH=${SPACK_ROOT}/.spack

    # Show spack info
    - spack debug report
  script:
    - source "${SPACK_VENV_DIR}/bin/activate"
    - source ${SPACK_ROOT}/share/spack/setup-env.sh

    # Add custom repo only if not already added
    - |
      if ! spack repo list | grep -q "${CI_PROJECT_DIR}/spack-repo"; then
        spack repo add ${CI_PROJECT_DIR}/spack-repo
      else
        echo "Repo already added: ${CI_PROJECT_DIR}/spack-repo"
      fi
    - spack repo list
    - spack list libra

    # Check if persistent base environment exists, create if not
    - |
      if spack env list | grep -q "^${SPACK_TEST_BASE_ENV}$"; then
        echo "Persistent base environment ${SPACK_TEST_BASE_ENV} already exists, reusing it"
      else
        echo "Creating persistent base environment ${SPACK_TEST_BASE_ENV}..."
        spack env create ${SPACK_TEST_BASE_ENV}
        spack env activate ${SPACK_TEST_BASE_ENV}
        
        # Add all dependencies to base environment (everything except libra)
        spack add kokkos+cuda+cuda_lambda+wrapper cuda_arch=70
        spack add casacore~python
        spack add cuda@12.2
        spack add fftw@3.3.10:+openmp
        spack add hpg@3.0.3:+cuda
        spack add parafeed@1.1.4:
        spack add py-pybind11@2.10.2:
        spack add googletest@1.14.0:
        spack add cmake@3.18:
        spack add pkgconfig
        spack add cfitsio
        spack add blas
        spack add lapack
        spack add boost
        spack add wcslib
        spack add readline
        spack add ncurses
        
        # Concretize and install base environment
        spack concretize -f
        spack install --fail-fast --verbose --no-checksum
        
        echo "Persistent base environment ${SPACK_TEST_BASE_ENV} created and installed"
      fi

    # Create job-specific libra test environment
    - spack env create ${LIBRA_TEST_ENV}
    - spack env activate ${LIBRA_TEST_ENV}
    
    # Add only libra to this environment with appropriate CUDA constraint
    - |
      GCC_VERSION=$(gcc --version | head -n1 | grep -oP 'gcc \(.*\) \K[0-9]+')
      if [ "$GCC_VERSION" = "8" ]; then
        echo "Using CUDA 12.2 constraint for GCC 8.x"
        spack add libra+cuda ^cuda@12.2
      else
        echo "Using latest CUDA for GCC $GCC_VERSION"
        spack add libra+cuda
      fi

    # Concretize and install libra (fresh build each time)
    - spack concretize -f
    - spack install --fail-fast --verbose --no-checksum --no-cache

    # Verify installation
    - spack find libra
    - spack find --deps libra
  after_script: |
    if [ -d "${SPACK_VENV_DIR}" ]; then
      source "${SPACK_VENV_DIR}/bin/activate"
      source ${SPACK_ROOT}/share/spack/setup-env.sh
    fi

    if [ "$CI_JOB_STATUS" = "success" ]; then
      echo "=== Build succeeded - cleaning up job-specific test environment ==="
      spack env remove --yes-to-all ${LIBRA_TEST_ENV} || true
      echo "Spack environment ${LIBRA_TEST_ENV} removed"
      echo "Persistent base environment ${SPACK_TEST_BASE_ENV} preserved for future runs"
    else
      echo "=== Build failed - preserving test environment for debugging ==="
      echo "Test environment preserved: ${LIBRA_TEST_ENV}"
      echo "To debug: source ${SPACK_ROOT}/share/spack/setup-env.sh && spack env activate ${LIBRA_TEST_ENV}"
    fi
  artifacts:
    when: always
    paths:
      - ${SPACK_CACHE_DIR}
    expire_in: 1 year
  timeout: 3h

# Manual jobs (simplified)
manual-trigger:
  stage: build
  before_script:
    # Ensure submodules are in sync with the index
    - git submodule sync
    - git submodule update --init --recursive
  script:
    - echo "Manual full build trigger initiated..."
    - export CCACHE_BASEDIR="$PWD"
    - export CCACHE_COMPILERCHECK=content
    - export CCACHE_MAXSIZE=2G
    - if [ -d "$CCACHE_DIR" ] && [ "$(ls -A $CCACHE_DIR)" ]; then echo "Using existing ccache"; else echo "No active ccache found"; fi
    - mkdir -p $CCACHE_DIR
    - ccache --show-stats || true
    - git clean -ffdx
    - mkdir -p build
    - cd build
    - cmake -DApps_BUILD_TESTS=ON -DKokkos_CUDA_ARCH_NAME=Kokkos_ARCH_VOLTA70 ..
    - make -j$(nproc)
    - ccache --show-stats || true
  when: manual
  only:
    - branches
  allow_failure: true

manual-cpu-only:
  stage: build
  rules:
    - when: manual
  allow_failure: true
  before_script:
    # Ensure submodules are in sync with the index
    - git submodule sync
    - git submodule update --init --recursive
    - export CCACHE_BASEDIR="$PWD"
    - export CCACHE_COMPILERCHECK=content
    - export CCACHE_MAXSIZE=5G
    - mkdir -p $CCACHE_DIR
    - if [ -d "$CCACHE_DIR" ] && [ "$(ls -A $CCACHE_DIR)" ]; then echo "Using existing ccache with $(ccache --show-stats | grep 'files in cache')"; else echo "Starting with empty ccache"; fi
    - ccache --show-stats || true
    - git clean -ffdx
  script:
    - echo "=== CPU-only build with LIBRA_DISABLE_CUDA=ON ==="
    - echo $PATH
    - echo "=== System Info (CPU-only build) ==="
    - nproc
    - echo "=== Building without CUDA ==="
    - mkdir -p build
    - cd build
    - cmake -DApps_BUILD_TESTS=ON -DNCORES=8 -DLIBRA_DISABLE_CUDA=ON ..
    - make -j$(nproc)
  after_script:
    - echo "=== ccache statistics ==="
    - ccache --show-stats || true
    - ccache --show-config || true

manual-spack:
  extends: spack-test
  when: manual
  rules:
    - when: manual
  allow_failure: true