From 9a0a552cccb88af856527e0fa82ef5f6430a10e9 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Fri, 19 Dec 2025 17:22:09 -0500 Subject: [PATCH] ci: fix frontier ci --- .gitignore | 1 + .gitlab/gitlab-ci-frontier.yml | 74 +++++++++++++++--------------- scripts/ci/gitlab-ci/run.sh | 70 +++++++++++----------------- scripts/ci/gitlab-ci/setup-vars.sh | 3 -- spack/configs/packages.yaml | 14 ++++++ 5 files changed, 79 insertions(+), 83 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/.gitlab/gitlab-ci-frontier.yml b/.gitlab/gitlab-ci-frontier.yml index e892656..e11ff87 100644 --- a/.gitlab/gitlab-ci-frontier.yml +++ b/.gitlab/gitlab-ci-frontier.yml @@ -8,7 +8,6 @@ default: stages: - pre - setup - - build - install - post @@ -30,13 +29,16 @@ stages: - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/ && $MAINTAINANCE_PIPELINE == null interruptible: true variables: + CI_BIN_DIR: "$CI_PROJECT_DIR/build" CUSTOM_CI_BUILDS_DIR: "/lustre/orion/ums032/scratch/ums032_auser/ci/tools-sdk/runtime" - OLCF_SERVICE_ACCOUNT: "ums032_auser" GIT_CONFIG_GLOBAL: "true" GITLAB_SITE: "OLCF Frontier" - CI_BIN_DIR: "$CI_PROJECT_DIR/build" - SPACK_CONFIG_FACILITY_DIR: "/lustre/orion/ums032/proj-shared/spack-configs/facility" - SPACK_CONFIG_DAV_SDK_DIR: "$CI_PROJECT_DIR/spack/configs" + NUM_CORES: "32" + OLCF_SERVICE_ACCOUNT: "ums032_auser" + SPACK_DISABLE_LOCAL_CONFIG: "true" + SPACK_BUILD_DIR: "${CUSTOM_CI_BUILDS_DIR}/artifacts/spack-build-${CI_PIPELINE_IID}" + SPACK_USER_CACHE_PATH: "${SPACK_BUILD_DIR}/cache" + # Order matters JOB_MODULES: >- Core/24.00 @@ -55,48 +57,34 @@ setup:frontier-gcc: - *checkout_commit - *load_modules script: + - source scripts/ci/gitlab-ci/setup-vars.sh - bash scripts/ci/gitlab-ci/run.sh setup - artifacts: - expire_in: 24 hours - when: always - paths: - - build/ - -build:frontier-gcc: - extends: .frontier-common - stage: build - tags: [shell] - before_script: - - *load_modules - script: - - bash scripts/ci/gitlab-ci/run.sh build - artifacts: - expire_in: 24 hours - when: always - paths: - - build/ - needs: - - setup:frontier-gcc - dependencies: - - setup:frontier-gcc install:frontier-gcc: extends: .frontier-common stage: install tags: [frontier, slurm] variables: - SCHEDULER_PARAMETERS: "-AUMS032 -pbatch -t 01:00:00 --nice=0 -c32 --threads-per-core=2 -N 1" + SCHEDULER_PARAMETERS: "-AUMS032 -pbatch -t 01:00:00 --nice=0 -c${NUM_CORES} --threads-per-core=2 -N 1 -C nvme" + SPACK_USER_CACHE_COMPUTING_PATH: "/mnt/bb/ums032_auser/spack-cache/" + SPACK_BUILD_COMPUTING_DIR: "/mnt/bb/ums032_auser/spack-build/" before_script: + - *checkout_commit - *load_modules script: - - bash scripts/ci/gitlab-ci/run.sh install + - source scripts/ci/gitlab-ci/setup-vars.sh + - ls /mnt/bb/ums032_auser/ + - rsync -av "${SPACK_BUILD_DIR}/spack" "${SPACK_BUILD_COMPUTING_DIR}" + - rsync -av "${SPACK_USER_CACHE_PATH}/" "${SPACK_USER_CACHE_COMPUTING_PATH}" + - SPACK_BUILD_DIR="${SPACK_BUILD_COMPUTING_DIR}" SPACK_USER_CACHE_PATH="${SPACK_USER_CACHE_COMPUTING_PATH}" bash scripts/ci/gitlab-ci/run.sh install after_script: - *load_modules + - source scripts/ci/gitlab-ci/setup-vars.sh - bash scripts/ci/gitlab-ci/run.sh submit needs: - - build:frontier-gcc + - setup:frontier-gcc dependencies: - - build:frontier-gcc + - setup:frontier-gcc .report-status: tags: [shell] @@ -127,7 +115,7 @@ success: variables: STATUS_DESC: Pipeline succeeded dependencies: - - build:frontier-gcc + - setup:frontier-gcc - install:frontier-gcc failure: @@ -139,7 +127,21 @@ failure: variables: STATUS_DESC: Pipeline failed dependencies: - - build:frontier-gcc + - setup:frontier-gcc + - install:frontier-gcc + +cleanup_spack: + stage: post + extends: .frontier-common + tags: [shell] + rules: + - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/ && $MAINTAINANCE_PIPELINE == null + when: always + script: + - source scripts/ci/gitlab-ci/setup-vars.sh + - rm -rf "${SPACK_BUILD_DIR}" + dependencies: + - setup:frontier-gcc - install:frontier-gcc generate_pipelines: @@ -169,7 +171,7 @@ generate_pipelines: - generated_pipelines.yml launch_pipelines: - stage: build + stage: post rules: - if: $CI_PIPELINE_SOURCE == "schedule" variables: @@ -183,7 +185,7 @@ launch_pipelines: # Frontier runtime files for the runners occasionally get corrupted, # Add manual job to remove it wipe_runtime_directory: - stage: build + stage: setup tags: [shell] rules: - if: $CI_PIPELINE_SOURCE =~ /web/ && $MAINTAINANCE_PIPELINE != null diff --git a/scripts/ci/gitlab-ci/run.sh b/scripts/ci/gitlab-ci/run.sh index 13fa71f..d42b8b9 100755 --- a/scripts/ci/gitlab-ci/run.sh +++ b/scripts/ci/gitlab-ci/run.sh @@ -7,9 +7,8 @@ die() { echo "error: $1"; exit "$2"; } while getopts "h" opt; do case $opt in h) - echo "Usage: $0 [setup|build|install|submit]" + echo "Usage: $0 [setup|install|submit]" echo " setup - Setup spack environment" - echo " build - Build (concretize) spack environment" echo " install - Install spack environment" echo " submit - Submit results (placeholder for CDash/reporting)" ;; @@ -24,29 +23,25 @@ readonly STEP=$1 [ -z "$STEP" ] && die "no argument given: $*" 2 shift -source scripts/ci/gitlab-ci/setup-vars.sh - # Determine which spack environment to use based on job name -ENV_NAME="frontier" +export TOOLS_SDK_ROOT=${PWD} + +if [ -z "${SPACK_BUILD_DIR}" ]; then + export SPACK_BUILD_DIR=${PWD}/build +fi + +: "${NUM_CORES:=4}" case ${STEP} in setup) echo "**********Setup Begin**********" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git "${SPACK_BUILD_DIR}/spack" - # Setup spack if not already available - if [ -z "$SPACK_ROOT" ]; then - if [ ! -d "${CI_ROOT_DIR}/spack" ]; then - git clone -c feature.manyFiles=true https://github.com/spack/spack.git "${CI_ROOT_DIR}/spack" - fi - # Source spack environment - . "${CI_ROOT_DIR}/spack/share/spack/setup-env.sh" - else - # Re-source existing spack installation - . "${SPACK_ROOT}/share/spack/setup-env.sh" - fi + # Source spack environment + . "${SPACK_BUILD_DIR}/spack/share/spack/setup-env.sh" # Activate the environment pointing to the config directory - spack env activate "${CI_SOURCE_DIR}/spack/configs/${ENV_NAME}" + spack env activate --create --without-view --envfile "${PWD}/spack/template" tools-sdk # Verify environment is active spack env status @@ -54,50 +49,37 @@ case ${STEP} in # Show configuration spack config blame - echo "**********Setup End**********" - ;; - - build) - echo "**********Build Begin**********" - - # Source spack - if [ -z "$SPACK_ROOT" ]; then - . "${CI_ROOT_DIR}/spack/share/spack/setup-env.sh" - else - . "${SPACK_ROOT}/share/spack/setup-env.sh" - fi - - # Activate the environment - spack env activate "${CI_SOURCE_DIR}/spack/configs/${ENV_NAME}" - # Concretize the environment spack concretize -f # Show what will be installed spack find - echo "**********Build End**********" + # Download archives + spack fetch --dependencies + + echo "**********Setup End**********" ;; install) echo "**********Install Begin**********" - # Source spack - if [ -z "$SPACK_ROOT" ]; then - . "${CI_ROOT_DIR}/spack/share/spack/setup-env.sh" - else - . "${SPACK_ROOT}/share/spack/setup-env.sh" - fi + . "${SPACK_BUILD_DIR}/spack/share/spack/setup-env.sh" + + # Activate the environment pointing to the config directory + spack env activate tools-sdk - # Activate the environment - spack env activate "${CI_SOURCE_DIR}/spack/configs/${ENV_NAME}" + # Verify environment is active + spack env status + + # Show configuration + spack config blame # Install the environment with timing and parallel jobs - spack -t install -j 8 --verbose --fail-fast + spack -t install "-j$((NUM_CORES*2))" --show-log-on-error --no-check-signature --fail-fast # Show what was installed spack find -lv - echo "**********Install End**********" ;; diff --git a/scripts/ci/gitlab-ci/setup-vars.sh b/scripts/ci/gitlab-ci/setup-vars.sh index a9732cb..ec7b715 100755 --- a/scripts/ci/gitlab-ci/setup-vars.sh +++ b/scripts/ci/gitlab-ci/setup-vars.sh @@ -1,9 +1,6 @@ #!/bin/bash --login set -e -# CI_ROOT_DIR: Parent directory where spack will be cloned -export CI_ROOT_DIR="${CI_PROJECT_DIR}/.." - # CI_SOURCE_DIR: Project source directory export CI_SOURCE_DIR="${CI_PROJECT_DIR}" diff --git a/spack/configs/packages.yaml b/spack/configs/packages.yaml index 7054379..34b3077 100644 --- a/spack/configs/packages.yaml +++ b/spack/configs/packages.yaml @@ -17,3 +17,17 @@ packages: conflict: - spec: +comm when: "@:2.34" + gcc: + externals: + - spec: gcc@12.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 + flags: {} + environment: + append_path: + PKG_CONFIG_PATH: /usr/lib64/pkgconfig + extra_rpaths: []