From bb9bd551aedbb4910604c946d183596c44c7f184 Mon Sep 17 00:00:00 2001 From: Shri Abhyankar Date: Sat, 31 Dec 2022 16:44:44 -0600 Subject: [PATCH 001/125] Added script to build GridPACK in one-go, including installing all dependencies. --- install_gridpack.sh | 123 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 install_gridpack.sh diff --git a/install_gridpack.sh b/install_gridpack.sh new file mode 100755 index 000000000..ffeee710b --- /dev/null +++ b/install_gridpack.sh @@ -0,0 +1,123 @@ +# This script installs GridPACK and all its dependencies.The dependencies are installed in external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. + +# This script should be run from the top-level GridPACK directory. + +# Set environment variable GRIDPACK_DIR +export GRIDPACK_DIR=${PWD} + +# Create directory for installing external packages +rm -rf external-dependencies +mkdir external-dependencies + +cd external-dependencies + +export GP_EXT_DEPS=${PWD} + +# Download and install Boost +echo "Downloading Boost-1.78.0" + +# Download Boost +wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + +# Untar +tar -xvf boost_1_78_0.tar.gz + +cd boost_1_78_0 + +# Install boost +echo "Building Boost-1.78.0" + +./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem + +cat project-config.jam >> `using mpi ;` + +./b2 -a -d+2 link=shared stage + +echo "Installing Boost-1.78.0" +./b2 -a -d+2 link=shared install + +echo "Building and Installing Boost libraries complete" + +# Download, build, and install GA +cd .. + +echo "Downloading GA-5.8" + +wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + +tar -xvf ga-5.8.tar.gz + +cd ga-5.8 + +# Build GA +echo "Building GA-5.8" +./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + +# Install GA +echo "Installing GA-5.8" +make -j 10 install + +echo "GA-5.8 installation complete" + + +# Install PETSc 3.16.4 +cd .. + +# Download +echo "Downloading PETSc 3.16.4" + +git clone https://gitlab.com/petsc/petsc.githttps://gitlab.com/petsc/petsc.git + +git checkout v3.16.4 + +cd petsc + +export PETSC_DIR=${PWD} +export PETSC_ARCH=build-dir + +# Install PETSc +echo "Installing PETSc 3.16.4" + +./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 --download-cmake=1 + +# Build PETSc +echo "Building PETSc 3.16.4" + +make + +# Install PETSc +echo "Installing PETSc 3.16.4" + +make install +make check + +## GridPACK installation +echo "Building GridPACK develop branch" + +git checkout develop + +cd ${GRIDPACK_DIR}/src + +rm -rf build +mkdir build + +cd build + +rm -rf CMake* +cmake \ + -D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ + -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ + -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ + -D BOOST_LIBRARYDIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_DIR}/src/install \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + .. + +echo "Installing GridPACK develop branch" +make -j 10 install From eb24be962688f233c74eb6b24196e961e6bd279d Mon Sep 17 00:00:00 2001 From: Shri Date: Thu, 16 Mar 2023 13:41:13 -0700 Subject: [PATCH 002/125] Updated installation script tested on EIOC-Ubuntu8 --- install_gridpack.sh | 211 +++++++++++++++++++++++++------------- src/lib/GridPACK.cmake.in | 8 +- 2 files changed, 144 insertions(+), 75 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index ffeee710b..9cc6ca4e8 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,123 +1,192 @@ -# This script installs GridPACK and all its dependencies.The dependencies are installed in external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. +# This script installs GridPACK and all its dependencies.The dependencies are installed in src/build/external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. -# Set environment variable GRIDPACK_DIR -export GRIDPACK_DIR=${PWD} +# Flags for installing/not installing different dependency packages +install_boost=true +install_ga=true +install_petsc=true -# Create directory for installing external packages -rm -rf external-dependencies -mkdir external-dependencies +# Flag for install GridPACK and GridPACK python wrapper +install_gridpack=true +install_gridpack_python=true -cd external-dependencies +# Set your python executable here +python_exe=`which python3` -export GP_EXT_DEPS=${PWD} +# Install log file +install_log_file=${PWD}/install.log -# Download and install Boost -echo "Downloading Boost-1.78.0" +echo 'GRIDPACK installation log file' > ${install_log_file} +echo $(date) >> install_log_file -# Download Boost -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz +export GRIDPACK_ROOT_DIR=${PWD} +# Set environment variable GRIDPACK_BUILD_DIR and create a build directory +export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build -# Untar -tar -xvf boost_1_78_0.tar.gz +rm -rf $GRIDPACK_BUILD_DIR +mkdir $GRIDPACK_BUILD_DIR +cd $GRIDPACK_BUILD_DIR -cd boost_1_78_0 +# Create directory for installing external packages +mkdir external-dependencies -# Install boost -echo "Building Boost-1.78.0" +export GP_EXT_DEPS=${GRIDPACK_BUILD_DIR}/external-dependencies -./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem +cd external-dependencies + +if ${install_boost} +then -cat project-config.jam >> `using mpi ;` + rm -rf boost* + + # Download and install Boost + echo "Downloading Boost-1.78.0" -./b2 -a -d+2 link=shared stage + # Download Boost + wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz &>> ${install_log_file} -echo "Installing Boost-1.78.0" -./b2 -a -d+2 link=shared install + # Untar + tar -xf boost_1_78_0.tar.gz -echo "Building and Installing Boost libraries complete" + cd boost_1_78_0 -# Download, build, and install GA -cd .. + # Install boost + echo "Building Boost-1.78.0" -echo "Downloading GA-5.8" + ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system &>> ${install_log_file} -wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + echo 'using mpi ;' >> project-config.jam -tar -xvf ga-5.8.tar.gz + ./b2 -a -d+2 link=shared stage &>> ${install_log_file} -cd ga-5.8 + echo "Installing Boost-1.78.0" + ./b2 -a -d+2 link=shared install &>> ${install_log_file} -# Build GA -echo "Building GA-5.8" -./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + echo "Building and Installing Boost libraries complete" +fi -# Install GA -echo "Installing GA-5.8" -make -j 10 install +if ${install_ga} +then + # Download, build, and install GA + cd ${GP_EXT_DEPS} -echo "GA-5.8 installation complete" + echo "Downloading GA-5.8" + wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz &>> ${install_log_file} -# Install PETSc 3.16.4 -cd .. + tar -xf ga-5.8.tar.gz &>> ${install_log_file} -# Download -echo "Downloading PETSc 3.16.4" + cd ga-5.8 -git clone https://gitlab.com/petsc/petsc.githttps://gitlab.com/petsc/petsc.git + # Build GA + echo "Building GA-5.8" + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared &>> ${install_log_file} + + # Install GA + echo "Installing GA-5.8" + make -j 10 install &>> ${install_log_file} + + echo "GA-5.8 installation complete" +fi -git checkout v3.16.4 +if ${install_petsc} +then + + # Install PETSc 3.16.4 + cd ${GP_EXT_DEPS} -cd petsc + # Download + echo "Downloading PETSc 3.16.4" -export PETSC_DIR=${PWD} -export PETSC_ARCH=build-dir + git clone https://gitlab.com/petsc/petsc.git &>> ${install_log_file} + + cd petsc -# Install PETSc -echo "Installing PETSc 3.16.4" + git checkout tags/v3.16.4 -b v3.16.4 &>> ${install_log_file} -./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 --download-cmake=1 + export PETSC_DIR=${PWD} + export PETSC_ARCH=build-dir -# Build PETSc -echo "Building PETSc 3.16.4" + # Install PETSc + echo "Installing PETSc 3.16.4" + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 &>> ${install_log_file} -make + # Build PETSc + echo "Building PETSc 3.16.4" -# Install PETSc -echo "Installing PETSc 3.16.4" + make &>> ${install_log_file} -make install -make check + # Install PETSc + echo "Installing PETSc 3.16.4" -## GridPACK installation -echo "Building GridPACK develop branch" + make install &>> ${install_log_file} + make check &>> ${install_log_file} +fi -git checkout develop +GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -cd ${GRIDPACK_DIR}/src +if ${install_gridpack} +then -rm -rf build -mkdir build + cd ${GRIDPACK_BUILD_DIR} + + ## GridPACK installation + echo "Building GridPACK develop branch" -cd build + git checkout develop -rm -rf CMake* -cmake \ - -D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ + rm -rf CMake* + + cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D BOOST_LIBRARYDIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D BOOST_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ -D MPIEXEC:STRING='mpiexec' \ -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_DIR}/src/install \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ - .. - -echo "Installing GridPACK develop branch" -make -j 10 install + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + .. " + + cmake ${cmake_args} + + echo "Installing GridPACK develop branch" + make -j 10 install +fi + +if ${install_gridpack_python} +then + echo "Installing GridPACK python wrapper" + + cd ${GRIDPACK_ROOT_DIR} + + git submodule update --init + + export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} + echo ${GRIDPACK_DIR} + cd python + + export RHEL_OPENMPI_HACK=yes + + ${python_exe} setup.py build + + rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python + mkdir ${GRIDPACK_INSTALL_DIR}/lib/python + + PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" + export PYTHONPATH + ${python_exe} setup.py install --home="$GRIDPACK_DIR" + +fi + +# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib +cd ${GRIDPACK_ROOT_DIR} + +echo "Completed GridPACK installation" diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index d351cd3f9..1e2516125 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -233,10 +233,10 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() - list(APPEND gp_include - @Boost_INCLUDE_DIRS@ - @MPI_INCLUDE_PATH@ - ) +# list(APPEND gp_include +# @Boost_INCLUDE_DIR@ +# @MPI_INCLUDE_PATH@ +# ) list(APPEND gp_libs @Boost_LIBRARIES@ From f29a18fc9d8e59be65f6b56f6c11d51cf1eb711e Mon Sep 17 00:00:00 2001 From: Shri Date: Thu, 16 Mar 2023 20:44:38 -0700 Subject: [PATCH 003/125] Minor modifications to install script --- install_gridpack.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 9cc6ca4e8..d4cd3482c 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -12,7 +12,11 @@ install_gridpack=true install_gridpack_python=true # Set your python executable here -python_exe=`which python3` +python_exe=`which python` +if[ -z ${python_exe}] +then + python_exe=`which python3` +fi # Install log file install_log_file=${PWD}/install.log @@ -126,6 +130,7 @@ then fi GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install +export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} if ${install_gridpack} then @@ -168,7 +173,6 @@ then git submodule update --init - export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} echo ${GRIDPACK_DIR} cd python From 42d480756d8d8b370bfbc2f322e40429e6aadcde Mon Sep 17 00:00:00 2001 From: Shri Abhyankar Date: Sat, 18 Mar 2023 10:16:34 -0700 Subject: [PATCH 004/125] Small modification to check python --- install_gridpack.sh | 2 +- src/ga | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index d4cd3482c..62665fa83 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,7 +13,7 @@ install_gridpack_python=true # Set your python executable here python_exe=`which python` -if[ -z ${python_exe}] +if test -z ${python_exe} then python_exe=`which python3` fi diff --git a/src/ga b/src/ga index cd638ca3b..fcdde0274 160000 --- a/src/ga +++ b/src/ga @@ -1 +1 @@ -Subproject commit cd638ca3b431566b29e55813095ae16a2da0bc47 +Subproject commit fcdde027422c256f42b852cd3d937242103da49d From 0e6b2cee6e7d05e879860129873d7be8c84c257c Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 14:32:05 -0700 Subject: [PATCH 005/125] Updated gridpack install script to only install GridPACK and python wrapper. --- install_gridpack.sh | 126 +++++--------------------------------------- src/ga | 2 +- 2 files changed, 14 insertions(+), 114 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 62665fa83..e0f6fc42e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,12 +1,7 @@ -# This script installs GridPACK and all its dependencies.The dependencies are installed in src/build/external-dependencies directory. GridPACK is built in src/build directory and installed in src/install directory. +# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true - # Flag for install GridPACK and GridPACK python wrapper install_gridpack=true install_gridpack_python=true @@ -18,116 +13,22 @@ then python_exe=`which python3` fi -# Install log file -install_log_file=${PWD}/install.log - -echo 'GRIDPACK installation log file' > ${install_log_file} -echo $(date) >> install_log_file - -export GRIDPACK_ROOT_DIR=${PWD} -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build - -rm -rf $GRIDPACK_BUILD_DIR -mkdir $GRIDPACK_BUILD_DIR -cd $GRIDPACK_BUILD_DIR - -# Create directory for installing external packages -mkdir external-dependencies - -export GP_EXT_DEPS=${GRIDPACK_BUILD_DIR}/external-dependencies - -cd external-dependencies - -if ${install_boost} +if test -z ${GRIDPACK_ROOT_DIR} then - - rm -rf boost* - - # Download and install Boost - echo "Downloading Boost-1.78.0" - - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz &>> ${install_log_file} - - # Untar - tar -xf boost_1_78_0.tar.gz - - cd boost_1_78_0 - - # Install boost - echo "Building Boost-1.78.0" - - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system &>> ${install_log_file} - - echo 'using mpi ;' >> project-config.jam - - ./b2 -a -d+2 link=shared stage &>> ${install_log_file} - - echo "Installing Boost-1.78.0" - ./b2 -a -d+2 link=shared install &>> ${install_log_file} - - echo "Building and Installing Boost libraries complete" + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi -if ${install_ga} +# Create directory for installing external packages +if test -z ${GP_EXT_DEPS} then - # Download, build, and install GA - cd ${GP_EXT_DEPS} - - echo "Downloading GA-5.8" - - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz &>> ${install_log_file} - - tar -xf ga-5.8.tar.gz &>> ${install_log_file} - - cd ga-5.8 - - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared &>> ${install_log_file} - - # Install GA - echo "Installing GA-5.8" - make -j 10 install &>> ${install_log_file} - - echo "GA-5.8 installation complete" + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies fi -if ${install_petsc} -then - - # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} - - # Download - echo "Downloading PETSc 3.16.4" - - git clone https://gitlab.com/petsc/petsc.git &>> ${install_log_file} - - cd petsc - - git checkout tags/v3.16.4 -b v3.16.4 &>> ${install_log_file} - - export PETSC_DIR=${PWD} - export PETSC_ARCH=build-dir - - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 &>> ${install_log_file} - - # Build PETSc - echo "Building PETSc 3.16.4" - - make &>> ${install_log_file} - - # Install PETSc - echo "Installing PETSc 3.16.4" +cd ${GRIDPACK_ROOT_DIR} - make install &>> ${install_log_file} - make check &>> ${install_log_file} -fi +# Set environment variable GRIDPACK_BUILD_DIR and create a build directory +export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} @@ -135,6 +36,9 @@ export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} if ${install_gridpack} then + rm -rf $GRIDPACK_BUILD_DIR + mkdir $GRIDPACK_BUILD_DIR + cd ${GRIDPACK_BUILD_DIR} ## GridPACK installation @@ -189,8 +93,4 @@ then fi -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib -cd ${GRIDPACK_ROOT_DIR} - echo "Completed GridPACK installation" diff --git a/src/ga b/src/ga index fcdde0274..cd638ca3b 160000 --- a/src/ga +++ b/src/ga @@ -1 +1 @@ -Subproject commit fcdde027422c256f42b852cd3d937242103da49d +Subproject commit cd638ca3b431566b29e55813095ae16a2da0bc47 From 2d9d2f18ed9a73e392bfe5de4f858a8147d32b86 Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 15:37:21 -0700 Subject: [PATCH 006/125] More work on installation script - Split the script into two (one for installing dependencies and other for GridPACK) - Ignore boost includes and boost libraries in GridPACK.cmake.in. Otherwise python installation fails. - Update GridPACK installation script with addition CMake flags for Boost. --- install_gridpack.sh | 15 +++-- install_gridpack_deps.sh | 126 ++++++++++++++++++++++++++++++++++++++ src/lib/GridPACK.cmake.in | 8 +-- 3 files changed, 141 insertions(+), 8 deletions(-) create mode 100755 install_gridpack_deps.sh diff --git a/install_gridpack.sh b/install_gridpack.sh index e0f6fc42e..933636dae 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -39,19 +39,22 @@ then rm -rf $GRIDPACK_BUILD_DIR mkdir $GRIDPACK_BUILD_DIR + rm -rf ${GRIDPACK_INSTALL_DIR} + cd ${GRIDPACK_BUILD_DIR} ## GridPACK installation - echo "Building GridPACK develop branch" + echo "Building GridPACK" - git checkout develop +# git checkout develop rm -rf CMake* cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D BOOST_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/build/lib \ + -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ + -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ + -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ -D MPI_CXX_COMPILER:STRING='mpicxx' \ -D MPI_C_COMPILER:STRING='mpicc' \ @@ -82,6 +85,8 @@ then export RHEL_OPENMPI_HACK=yes + rm -rf build + ${python_exe} setup.py build rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python @@ -93,4 +98,6 @@ then fi +cd ${GRIDPACK_ROOT_DIR} + echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh new file mode 100755 index 000000000..e6e054e69 --- /dev/null +++ b/install_gridpack_deps.sh @@ -0,0 +1,126 @@ +# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. + +# This script should be run from the top-level GridPACK directory. + +# Flags for installing/not installing different dependency packages +install_boost=true +install_ga=true +install_petsc=true + +echo $(date) + +if test -z ${GRIDPACK_ROOT_DIR} +then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +fi + +cd ${GRIDPACK_ROOT_DIR} + +# Create directory for installing external packages +if test -z ${GP_EXT_DEPS} +then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies + rm -rf ${GP_EXT_DEPS} + mkdir ${GP_EXT_DEPS} + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +fi + +cd ${GP_EXT_DEPS} + +if ${install_boost} +then + + rm -rf boost* + + # Download and install Boost + echo "Downloading Boost-1.78.0" + + # Download Boost + wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + + # Untar + tar -xf boost_1_78_0.tar.gz + + cd boost_1_78_0 + + # Install boost + echo "Building Boost-1.78.0" + + ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system + + echo 'using mpi ;' >> project-config.jam + + ./b2 -a -d+2 link=shared stage + + echo "Installing Boost-1.78.0" + ./b2 -a -d+2 link=shared install + + echo "Building and Installing Boost libraries complete" +fi + +if ${install_ga} +then + # Download, build, and install GA + cd ${GP_EXT_DEPS} + + echo "Downloading GA-5.8" + + wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + + tar -xf ga-5.8.tar.gz + + cd ga-5.8 + + # Build GA + echo "Building GA-5.8" + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared + + # Install GA + echo "Installing GA-5.8" + make -j 10 install + + echo "GA-5.8 installation complete" +fi + +if ${install_petsc} +then + + # Install PETSc 3.16.4 + cd ${GP_EXT_DEPS} + + # Download + echo "Downloading PETSc 3.16.4" + + git clone https://gitlab.com/petsc/petsc.git + + cd petsc + + git checkout tags/v3.16.4 -b v3.16.4 + + export PETSC_DIR=${PWD} + export PETSC_ARCH=build-dir + + # Install PETSc + echo "Installing PETSc 3.16.4" + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 + + # Build PETSc + echo "Building PETSc 3.16.4" + + make + + # Install PETSc + echo "Installing PETSc 3.16.4" + + make install + make check +fi + +# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it +export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + +cd ${GRIDPACK_ROOT_DIR} + +echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 1e2516125..eaa59dc56 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -238,10 +238,10 @@ function(gridpack_setup) # @MPI_INCLUDE_PATH@ # ) - list(APPEND gp_libs - @Boost_LIBRARIES@ - @MPI_CXX_LIBRARIES@ - ) +# list(APPEND gp_libs +# @Boost_LIBRARIES@ +# @MPI_CXX_LIBRARIES@ +# ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 177881703a06763a8b711345d5b587b8546e1612 Mon Sep 17 00:00:00 2001 From: Shri Date: Fri, 7 Apr 2023 16:27:07 -0700 Subject: [PATCH 007/125] Create external-dependencies directory, if not present. --- install_gridpack_deps.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index e6e054e69..87897593e 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -23,7 +23,14 @@ then export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies rm -rf ${GP_EXT_DEPS} mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +else + if test -d ${GP_EXT_DEPS} + then + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + else + mkdir ${GP_EXT_DEPS} + fi fi cd ${GP_EXT_DEPS} From 37c8dd77b4e0aa3c344b575f45fba2deb9e60c30 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 5 Sep 2023 21:44:01 -0500 Subject: [PATCH 008/125] Apply shellcheck suggestions and run formatter --- install_gridpack.sh | 111 +++++++++++++++++++-------------------- install_gridpack_deps.sh | 82 ++++++++++++++--------------- 2 files changed, 93 insertions(+), 100 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 933636dae..523adef1c 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,3 +1,5 @@ +#! /bin/bash + # This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. # This script should be run from the top-level GridPACK directory. @@ -7,25 +9,22 @@ install_gridpack=true install_gridpack_python=true # Set your python executable here -python_exe=`which python` -if test -z ${python_exe} -then - python_exe=`which python3` +python_exe=$(which python) +if test -z "${python_exe}"; then + python_exe=$(which python3) fi -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +if test -z "${GRIDPACK_ROOT_DIR}"; then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi # Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies +if test -z "${GP_EXT_DEPS}"; then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit # Set environment variable GRIDPACK_BUILD_DIR and create a build directory export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build @@ -33,71 +32,69 @@ export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} -if ${install_gridpack} -then +if ${install_gridpack}; then + + rm -rf "$GRIDPACK_BUILD_DIR" + mkdir "$GRIDPACK_BUILD_DIR" - rm -rf $GRIDPACK_BUILD_DIR - mkdir $GRIDPACK_BUILD_DIR + rm -rf "${GRIDPACK_INSTALL_DIR}" - rm -rf ${GRIDPACK_INSTALL_DIR} + cd "${GRIDPACK_BUILD_DIR}" || exit - cd ${GRIDPACK_BUILD_DIR} - ## GridPACK installation echo "Building GridPACK" -# git checkout develop + # git checkout develop rm -rf CMake* cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ + -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ + -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ + -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ .. " - - cmake ${cmake_args} - echo "Installing GridPACK develop branch" - make -j 10 install + cmake "${cmake_args}" + + echo "Installing GridPACK develop branch" + make -j 10 install fi -if ${install_gridpack_python} -then - echo "Installing GridPACK python wrapper" - - cd ${GRIDPACK_ROOT_DIR} - - git submodule update --init - - echo ${GRIDPACK_DIR} - cd python - - export RHEL_OPENMPI_HACK=yes - - rm -rf build - - ${python_exe} setup.py build - - rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python - mkdir ${GRIDPACK_INSTALL_DIR}/lib/python - - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH - ${python_exe} setup.py install --home="$GRIDPACK_DIR" - +if ${install_gridpack_python}; then + echo "Installing GridPACK python wrapper" + + cd "${GRIDPACK_ROOT_DIR}" || exit + + git submodule update --init + + echo "${GRIDPACK_DIR}" + cd python || exit + + export RHEL_OPENMPI_HACK=yes + + rm -rf build + + ${python_exe} setup.py build + + rm -rf "${GRIDPACK_INSTALL_DIR}"/lib/python + mkdir "${GRIDPACK_INSTALL_DIR}"/lib/python + + PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" + export PYTHONPATH + ${python_exe} setup.py install --home="$GRIDPACK_DIR" + fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 87897593e..ed54e5d3d 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -1,3 +1,5 @@ +#! /bin/bash + # This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. # This script should be run from the top-level GridPACK directory. @@ -7,39 +9,35 @@ install_boost=true install_ga=true install_petsc=true -echo $(date) +date -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" +if test -z "${GRIDPACK_ROOT_DIR}"; then + export GRIDPACK_ROOT_DIR=${PWD} + echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" fi -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit # Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf ${GP_EXT_DEPS} - mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" +if test -z "${GP_EXT_DEPS}"; then + export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies + rm -rf "${GP_EXT_DEPS}" + mkdir "${GP_EXT_DEPS}" + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" else - if test -d ${GP_EXT_DEPS} - then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir ${GP_EXT_DEPS} - fi + if test -d "${GP_EXT_DEPS}"; then + echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" + else + mkdir "${GP_EXT_DEPS}" + fi fi -cd ${GP_EXT_DEPS} +cd "${GP_EXT_DEPS}" || exit -if ${install_boost} -then +if ${install_boost}; then rm -rf boost* - + # Download and install Boost echo "Downloading Boost-1.78.0" @@ -49,14 +47,14 @@ then # Untar tar -xf boost_1_78_0.tar.gz - cd boost_1_78_0 + cd boost_1_78_0 || exit # Install boost echo "Building Boost-1.78.0" ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system - echo 'using mpi ;' >> project-config.jam + echo 'using mpi ;' >>project-config.jam ./b2 -a -d+2 link=shared stage @@ -66,10 +64,9 @@ then echo "Building and Installing Boost libraries complete" fi -if ${install_ga} -then +if ${install_ga}; then # Download, build, and install GA - cd ${GP_EXT_DEPS} + cd "${GP_EXT_DEPS}" || exit echo "Downloading GA-5.8" @@ -77,31 +74,30 @@ then tar -xf ga-5.8.tar.gz - cd ga-5.8 + cd ga-5.8 || exit # Build GA echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared - + ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix="${PWD}"/install_for_gridpack --enable-shared + # Install GA echo "Installing GA-5.8" make -j 10 install - + echo "GA-5.8 installation complete" fi -if ${install_petsc} -then - +if ${install_petsc}; then + # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} + cd "${GP_EXT_DEPS}" || exit # Download echo "Downloading PETSc 3.16.4" git clone https://gitlab.com/petsc/petsc.git - - cd petsc + + cd petsc || exit git checkout tags/v3.16.4 -b v3.16.4 @@ -110,24 +106,24 @@ then # Install PETSc echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 + + ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix="${PWD}"/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 # Build PETSc echo "Building PETSc 3.16.4" - make + make # Install PETSc echo "Installing PETSc 3.16.4" make install - make check -fi + make check +fi # update LD_LIBRARY_PATH so that boost,ga, and petsc are on it export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} -cd ${GRIDPACK_ROOT_DIR} +cd "${GRIDPACK_ROOT_DIR}" || exit echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" From 24cdd96196a09350e63e202b34e28cbd519bf63e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 5 Sep 2023 22:41:14 -0500 Subject: [PATCH 009/125] Move blocks into functions and extract versions - Remove directory changes and assume current directory - Use local variables in functions - Capture logs --- install_gridpack.sh | 153 ++++++++++++++++------------- install_gridpack_deps.sh | 207 ++++++++++++++++++++++----------------- 2 files changed, 203 insertions(+), 157 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 523adef1c..21d676270 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,100 +1,115 @@ #! /bin/bash -# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. +# installs GridPACK and its python wrappter +# gridPACK is built in src/build and installed to src/install +# run this from the top level GridPACK directory -# This script should be run from the top-level GridPACK directory. +set -xeuo pipefail -# Flag for install GridPACK and GridPACK python wrapper -install_gridpack=true -install_gridpack_python=true +function install_gridpack { + echo "--- GridPACK ---" -# Set your python executable here -python_exe=$(which python) -if test -z "${python_exe}"; then - python_exe=$(which python3) -fi + # args + local gridpack_deps_dir=$1 + local gridpack_build_dir=$2 + local gridpack_install_dir=$3 + ${gridpack_deps_dir:?} + ${gridpack_build_dir:?} + ${gridpack_install_dir:?} -if test -z "${GRIDPACK_ROOT_DIR}"; then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi + # remove existing build and install dir + rm -rf "$gridpack_build_dir" + rm -rf "$gridpack_install_dir" -# Create directory for installing external packages -if test -z "${GP_EXT_DEPS}"; then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies -fi + # create the build dir + mkdir -p "$gridpack_build_dir" -cd "${GRIDPACK_ROOT_DIR}" || exit + pushd "$gridpack_build_dir" || exit -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build - -GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} - -if ${install_gridpack}; then - - rm -rf "$GRIDPACK_BUILD_DIR" - mkdir "$GRIDPACK_BUILD_DIR" - - rm -rf "${GRIDPACK_INSTALL_DIR}" - - cd "${GRIDPACK_BUILD_DIR}" || exit - - ## GridPACK installation - echo "Building GridPACK" - - # git checkout develop + # todo? git checkout develop + # remove existing cmake output rm -rf CMake* - cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ - -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ - -D CMAKE_BUILD_TYPE:STRING=Debug \ - -D BUILD_SHARED_LIBS=YES \ - -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ - .. " - - cmake "${cmake_args}" - - echo "Installing GridPACK develop branch" + # generate make files + echo "Generating GridPACK make files" + cmake \ + -D GA_DIR:STRING="${gridpack_deps_dir}/ga/install_for_gridpack" \ + -D BOOST_ROOT:STRING="${gridpack_deps_dir}/boost/install_for_gridpack" \ + -D Boost_DIR:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib/cmake/Boost" \ + -D Boost_LIBRARIES:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib" \ + -D Boost_INCLUDE_DIRS:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/include" \ + -D PETSC_DIR:PATH="${gridpack_deps_dir}/petsc/install_for_gridpack" \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + .. + + # install + echo "Installing GridPACK" make -j 10 install -fi -if ${install_gridpack_python}; then - echo "Installing GridPACK python wrapper" + popd || exit - cd "${GRIDPACK_ROOT_DIR}" || exit + echo "GridPACK installation complete" +} +function install_gridpack_python { + echo "--- GridPACK python wrapper ---" + + # args + local gridpack_build_dir=$1 + local gridpack_install_dir=$2 + ${gridpack_build_dir:?} + ${gridpack_install_dir:?} + + # update submodules + echo "Updating GridPACK submodules" git submodule update --init - echo "${GRIDPACK_DIR}" - cd python || exit + pushd python || exit export RHEL_OPENMPI_HACK=yes + # set python executable path + python_exe=$(which python || which python3) + + # remove existing build dir rm -rf build + # build + echo "Building GridPACK python wrapper" ${python_exe} setup.py build - rm -rf "${GRIDPACK_INSTALL_DIR}"/lib/python - mkdir "${GRIDPACK_INSTALL_DIR}"/lib/python + # remove existing install dir + local py_lib="${gridpack_install_dir}/lib/python" + rm -rf "${py_lib}" + mkdir -p "${py_lib}" + + # add lib to python path + export PYTHONPATH="${py_lib}:${PYTHONPATH}" - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH + # install + echo "Installing GridPACK python wrapper" ${python_exe} setup.py install --home="$GRIDPACK_DIR" -fi + popd || exit + + echo "GridPACK python wrapper installation complete" +} + +echo "Installing GridPACK" +date + +build_dir=${PWD}/src/build +install_dir=${PWD}/src/install -cd "${GRIDPACK_ROOT_DIR}" || exit +install_gridpack "${GRIDPACK_EXT_DEPS-?}" "$build_dir" "$install_dir" +install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index ed54e5d3d..94b55000f 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -1,129 +1,160 @@ #! /bin/bash -# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. +# installs GridPACK dependencies to the current directory -# This script should be run from the top-level GridPACK directory. +set -xeuo pipefail -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true +function install_boost { + local boost_version=$1 -date - -if test -z "${GRIDPACK_ROOT_DIR}"; then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi + # check args + : "${boost_version:?}" -cd "${GRIDPACK_ROOT_DIR}" || exit + echo "--- Installing Boost ${boost_version} ---" -# Create directory for installing external packages -if test -z "${GP_EXT_DEPS}"; then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf "${GP_EXT_DEPS}" - mkdir "${GP_EXT_DEPS}" - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" -else - if test -d "${GP_EXT_DEPS}"; then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir "${GP_EXT_DEPS}" - fi -fi + # remove existing + rm -rf boost* -cd "${GP_EXT_DEPS}" || exit + # download + echo "Downloading Boost" + wget \ + "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ + -O boost.tar.gz \ + --quiet -if ${install_boost}; then + # unpack + echo "Unpacking Boost" + tar -xf boost.tar.gz && rm boost.tar.gz - rm -rf boost* + # remove version from dir name + mv "boost_${boost_version//./_}" boost - # Download and install Boost - echo "Downloading Boost-1.78.0" + pushd boost || exit - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz - - # Untar - tar -xf boost_1_78_0.tar.gz + # bootstrap + echo "Bootstrapping Boost" + ./bootstrap.sh \ + --prefix=install_for_gridpack \ + --with-libraries=mpi,serialization,random,filesystem,system \ + >../log/boost_bootstrap.log 2>&1 + echo 'using mpi ;' >>project-config.jam - cd boost_1_78_0 || exit + # build + echo "Building Boost" + ./b2 -a -d+2 link=shared stage >../log/boost_build.log 2>&1 - # Install boost - echo "Building Boost-1.78.0" + # install + echo "Installing Boost" + ./b2 -a -d+2 link=shared install >../log/boost_install.log 2>&1 - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system + popd || exit - echo 'using mpi ;' >>project-config.jam + echo "Boost installation complete" +} - ./b2 -a -d+2 link=shared stage +function install_ga { + local ga_version=$1 - echo "Installing Boost-1.78.0" - ./b2 -a -d+2 link=shared install + # check args + : "${ga_version:?}" - echo "Building and Installing Boost libraries complete" -fi + echo "--- Installing Global Arrays ${ga_version} ---" -if ${install_ga}; then - # Download, build, and install GA - cd "${GP_EXT_DEPS}" || exit + # download + echo "Downloading Global Arrays" + wget \ + "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ + -O ga.tar.gz \ + --quiet - echo "Downloading GA-5.8" + # unpack + echo "Unpacking Global Arrays" + tar -xf ga.tar.gz && rm ga.tar.gz - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + # remove version from dir name + mv "ga-${ga_version}" ga - tar -xf ga-5.8.tar.gz + pushd ga || exit - cd ga-5.8 || exit + # build + echo "Configuring Global Arrays" + ./configure \ + --with-mpi-ts \ + --disable-f77 \ + --without-blas \ + --enable-cxx \ + --enable-i4 \ + --prefix="${PWD}/install_for_gridpack" \ + --enable-shared \ + >../log/ga_configure.log 2>&1 - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix="${PWD}"/install_for_gridpack --enable-shared + # install + echo "Installing Global Arrays" + make -j 10 install >../log/ga_install.log 2>&1 - # Install GA - echo "Installing GA-5.8" - make -j 10 install + popd || exit - echo "GA-5.8 installation complete" -fi + echo "Global Arrays installation complete" +} -if ${install_petsc}; then +function install_petsc { + local petsc_version=$1 - # Install PETSc 3.16.4 - cd "${GP_EXT_DEPS}" || exit + # check args + : "${petsc_version:?}" - # Download - echo "Downloading PETSc 3.16.4" + echo "--- Installing PETSc ${petsc_version} ---" + # clone + echo "Cloning PETSc repository" git clone https://gitlab.com/petsc/petsc.git - cd petsc || exit + pushd petsc || exit - git checkout tags/v3.16.4 -b v3.16.4 + git checkout "tags/v${petsc_version}" -b "v${petsc_version}" export PETSC_DIR=${PWD} export PETSC_ARCH=build-dir - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix="${PWD}"/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 - - # Build PETSc - echo "Building PETSc 3.16.4" - - make - - # Install PETSc - echo "Installing PETSc 3.16.4" - - make install - make check -fi + # install + echo "Configuring PETSc" + ./configure \ + --download-superlu_dist \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --download-cmake \ + --prefix="${PWD}"/install_for_gridpack \ + --scalar-type=complex \ + --with-shared-libraries=1 \ + --download-f2cblaslapack=1 \ + >../log/petsc_configure.log 2>&1 + + # build + echo "Building PETSc" + make >../log/petsc_build.log 2>&1 + + # install + echo "Installing PETSc" + make install >../log/petsc_install.log 2>&1 + + # check + echo "Checking PETSc" + make check >../log/petsc_check.log 2>&1 + + popd || exit + + echo "PETSc installation complete" +} + +echo "Installing GridPACK dependencies" +date -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +mkdir -p log -cd "${GRIDPACK_ROOT_DIR}" || exit +install_boost "1.78.0" +install_ga "5.8" +install_petsc "3.16.4" -echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" +echo "GridPACK dependency installation complete" From af2952c568eec49a1c6dab61c1f1de3f851d6a6b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 01:39:00 -0500 Subject: [PATCH 010/125] Correct how variables are checked --- install_gridpack.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 21d676270..fdadabf24 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,9 +13,7 @@ function install_gridpack { local gridpack_deps_dir=$1 local gridpack_build_dir=$2 local gridpack_install_dir=$3 - ${gridpack_deps_dir:?} - ${gridpack_build_dir:?} - ${gridpack_install_dir:?} + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -65,8 +63,7 @@ function install_gridpack_python { # args local gridpack_build_dir=$1 local gridpack_install_dir=$2 - ${gridpack_build_dir:?} - ${gridpack_install_dir:?} + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" # update submodules echo "Updating GridPACK submodules" @@ -109,7 +106,7 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GRIDPACK_EXT_DEPS-?}" "$build_dir" "$install_dir" +install_gridpack "${GRIDPACK_EXT_DEPS:?}" "$build_dir" "$install_dir" install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" From 1e9392431984a8399759ee6546fcf38ea16bda3c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 01:39:37 -0500 Subject: [PATCH 011/125] Bump versions out of script with env vars --- install_gridpack_deps.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 94b55000f..74a5dd15a 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -153,8 +153,8 @@ date mkdir -p log -install_boost "1.78.0" -install_ga "5.8" -install_petsc "3.16.4" +install_boost "${BOOST_VERSION:?}" +install_ga "${GA_VERSION:?}" +install_petsc "${PETSC_VERSION:?}" echo "GridPACK dependency installation complete" From 1d509f05da29b5a10b451e5616e7d1a903630d5d Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 23 Aug 2023 14:24:13 -0700 Subject: [PATCH 012/125] Remove GA-based dense matrix; adjust matrix transpose --- src/example_configuration.sh | 21 +- src/math/CMakeLists.txt | 12 +- src/math/petsc/ga_matrix.cpp | 903 --------------------- src/math/petsc/ga_matrix.hpp | 69 -- src/math/petsc/petsc_matrix_operations.cpp | 58 +- src/math/test/petsc_ga_matrix.cpp | 358 -------- 6 files changed, 16 insertions(+), 1405 deletions(-) delete mode 100644 src/math/petsc/ga_matrix.cpp delete mode 100644 src/math/petsc/ga_matrix.hpp delete mode 100644 src/math/test/petsc_ga_matrix.cpp diff --git a/src/example_configuration.sh b/src/example_configuration.sh index 0e0cb3cc9..ee870d678 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -265,30 +265,19 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ # -D USE_OLD_PETSC:BOOL=OFF \ - # System PETSc package: - # -D PETSC_DIR:STRING="/usr/lib/petsc" \ - # -D PETSC_ARCH:STRING="" \ + # Custom built 3.14, complex: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - + prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ - # Custom built 3.12.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.12.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.12" \ - # -D USE_OLD_PETSC:BOOL=OFF \ - - # Custom built 3.10.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.10.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.10" \ - # -D USE_OLD_PETSC:BOOL=OFF \ - - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ + -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ diff --git a/src/math/CMakeLists.txt b/src/math/CMakeLists.txt index 5146dc957..b7220c312 100644 --- a/src/math/CMakeLists.txt +++ b/src/math/CMakeLists.txt @@ -7,7 +7,7 @@ # distribution. # ------------------------------------------------------------- # Created May 7, 2013 by William A. Perkins -# Last Change: 2022-10-05 08:15:59 d3g096 +# Last Change: 2023-08-23 12:35:13 d3g096 # ------------------------------------------------------------- set(gridpack_math_sources) set(target_libraries @@ -80,7 +80,6 @@ if (PETSC_FOUND) petsc/petsc_vector.cpp petsc/petsc_vector_wrapper.cpp petsc/petsc_matrix.cpp - petsc/ga_matrix.cpp petsc/petsc_matrix_operations.cpp petsc/petsc_matrix_wrapper.cpp petsc/petsc_matrix.cpp @@ -359,15 +358,6 @@ add_executable(bouncing_ball_test test/bouncing_ball_test.cpp) target_link_libraries(bouncing_ball_test gridpack_math ${target_libraries}) -# ------------------------------------------------------------- -# Test GA-based Dense Matrix -# ------------------------------------------------------------- -if (PETSC_FOUND) - add_executable(petsc_ga_matrix_test test/petsc_ga_matrix.cpp) - target_link_libraries(petsc_ga_matrix_test gridpack_math ${target_libraries}) - gridpack_add_unit_test(petsc_ga_matrix petsc_ga_matrix_test) -endif() - # ------------------------------------------------------------- # installation # ------------------------------------------------------------- diff --git a/src/math/petsc/ga_matrix.cpp b/src/math/petsc/ga_matrix.cpp deleted file mode 100644 index 7e4cf6e6b..000000000 --- a/src/math/petsc/ga_matrix.cpp +++ /dev/null @@ -1,903 +0,0 @@ -/* ------------------------------------------------------------- - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - * ------------------------------------------------------------- */ -/** - * @file ga_matrix.c - * @author William A. Perkins - * @date 2019-07-29 12:16:09 d3g096 - * - * @brief - * - * - */ - - -#include -#include -#include - -#include -#include "ga_matrix.hpp" - - -// ------------------------------------------------------------- -// Determine the Global Arrays data type from PETSc configuration -// ------------------------------------------------------------- -#if defined(PETSC_USE_REAL_DOUBLE) - -#if defined(PETSC_USE_COMPLEX) -typedef DoubleComplex PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_DCPL - -#else -typedef double PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_DBL -#endif - -#else - -#if defined(PETSC_USE_COMPLEX) -typedef SingleComplex PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_SCPL; -#else -typedef float PetscScalarGA; -#define MT_PETSC_SCALAR MT_C_FLOAT -#endif - -#endif - -static const PetscScalarGA one = -#if defined(PETSC_USE_COMPLEX) - { 1.0, 0.0 } -#else - 1.0 -#endif - ; - -static const PetscScalarGA zero = -#if defined(PETSC_USE_COMPLEX) - { 0.0, 0.0 } -#else - 0.0 -#endif - ; - - - -// ------------------------------------------------------------- -// struct MatGACtx -// ------------------------------------------------------------- -struct MatGACtx { - int gaGroup; /**< GA process group handle */ - int ga; /**< GA handle */ - int lo[2]; /**< Lower limits of "local" matrix ownership */ - int hi[2]; /**< Upper limits of "local" matrix ownership */ -}; - -static PetscErrorCode MatSetOperations_DenseGA(Mat A); - -// ------------------------------------------------------------- -// CreateMatGA -// ------------------------------------------------------------- -static -PetscErrorCode -CreateMatGA(int pgroup, int lrows, int lcols, int grows, int gcols, int *ga) -{ - PetscErrorCode ierr = 0; - - /* Try to honor local ownership request (of rows). */ - - int nprocs = GA_Pgroup_nnodes(pgroup); - int me = GA_Pgroup_nodeid(pgroup); - std::vector tmapc(nprocs+1); - std::vector mapc(nprocs+1); - int i; - - for (i = 0; i < nprocs+1; i++) tmapc[i] = 0; - tmapc[me] = lrows; - GA_Pgroup_igop(pgroup, &tmapc[0], nprocs+1, "+"); - mapc[0] = 0; - for (i = 1; i < nprocs; i++) mapc[i] = mapc[i-1]+tmapc[i-1]; - mapc[nprocs] = 0; - - int dims[2] = {grows, gcols}; - int blocks[2] = { nprocs, 1 }; - - - *ga = NGA_Create_irreg_config(MT_PETSC_SCALAR, 2, &dims[0], - "PETSc Matrix in GA", - &blocks[0], &mapc[0], - pgroup); - - PetscScalar z(0.0); - GA_Fill(*ga, &z); - - return ierr; -} - -// ------------------------------------------------------------- -// MPIComm2GApgroup -// ------------------------------------------------------------- -static -PetscErrorCode -MPIComm2GApgroup(MPI_Comm comm, int *pGrpHandle) -{ - PetscErrorCode ierr = 0; - int nproc; - int me, myGlobalRank; - int *proclist; - int p; - - ierr = MPI_Comm_size(comm, &nproc); CHKERRQ(ierr); - ierr = MPI_Comm_rank(comm, &me); CHKERRQ(ierr); - myGlobalRank = GA_Nodeid(); - ierr = PetscMalloc(nproc*sizeof(int), &proclist); CHKERRQ(ierr); - for (p = 0; p < nproc; ++p) { - proclist[p] = 0; - } - proclist[me] = myGlobalRank; - ierr = MPI_Allreduce(MPI_IN_PLACE, &proclist[0], nproc, MPI_INT, MPI_SUM, comm); CHKERRQ(ierr); - *pGrpHandle = GA_Pgroup_create(&proclist[0], nproc); - ierr = PetscFree(proclist); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// Mat2GA -// This should only be called by MatMultbyGA(). It assumes the Mat is -// dense and the GA has the same ownership distribution. -// ------------------------------------------------------------- -static -PetscErrorCode -Mat2GA(const Mat& A, int Aga) -{ - PetscErrorCode ierr(0); - int me(GA_Nodeid()); - int r, rmin, rmax; - int lo[2], hi[2], ld[2]; - int i, j; - PetscInt grows, gcols, lrows, lcols; - PetscScalar *matdata; - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetOwnershipRange(A, &rmin, &rmax); CHKERRQ(ierr); - - NGA_Distribution(Aga, me, &lo[0], &hi[0]); - - std::vector cidx(gcols); - for (j = 0; j < gcols; ++j) cidx[j] = j; - std::vector rdata(gcols); - PetscScalarGA *gadata; - - for (r = rmin; r < rmax; ++r) { - ierr = MatGetValues(A, 1, &r, gcols, &cidx[0], &rdata[0]); CHKERRQ(ierr); - lo[0] = r; hi[0] = r; - lo[1] = 0; hi[1] = gcols - 1; - ld[0] = 1; - NGA_Access(Aga, &lo[0], &hi[0], &gadata, &ld[0]); - for (j = 0; j < gcols; ++j) { -#if defined(PETSC_USE_COMPLEX) - gadata[j].real = rdata[j].real(); - gadata[j].imag = rdata[j].imag(); -#else - gadata[j] = rdata[j]; -#endif - } - NGA_Release_update(Aga, &lo[0], &hi[0]); - } - GA_Sync(); - // GA_Print(Aga); - return ierr; -} - -// ------------------------------------------------------------- -// GA2Mat -// This should only be called by MatMultbyGA(). It assumes the Mat is -// dense and the GA has the same ownership distribution. -// ------------------------------------------------------------- -static -PetscErrorCode -GA2Mat(const int Aga, Mat& A) -{ - PetscErrorCode ierr(0); - int me(GA_Nodeid()); - int r, rmin, rmax; - int lo[2], hi[2], ld[2]; - int i, j; - PetscInt grows, gcols, lrows, lcols; - PetscScalar *matdata; - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetOwnershipRange(A, &rmin, &rmax); CHKERRQ(ierr); - - NGA_Distribution(Aga, me, &lo[0], &hi[0]); - - std::vector cidx(gcols); - for (j = 0; j < gcols; ++j) cidx[j] = j; - std::vector rdata(gcols); - PetscScalarGA *gadata; - - for (r = rmin; r < rmax; ++r) { - lo[0] = r; hi[0] = r; - lo[1] = 0; hi[1] = gcols - 1; - ld[0] = 1; - NGA_Access(Aga, &lo[0], &hi[0], &gadata, &ld[0]); - for (j = 0; j < gcols; ++j) { -#if defined(PETSC_USE_COMPLEX) - rdata[j].real(gadata[j].real); - rdata[j].imag(gadata[j].imag); -#else - rdata[j] = gadata[j]; -#endif - } - ierr = MatSetValues(A, 1, &r, gcols, &cidx[0], &rdata[0], INSERT_VALUES); CHKERRQ(ierr); - NGA_Release(Aga, &lo[0], &hi[0]); - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMultbyGA -// ------------------------------------------------------------- -PetscErrorCode -MatMultbyGA(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - MPI_Comm comm; - int lrows, grows, lcols, gcols; - int pgp, opgp; - int Aga, Bga, Cga; - int m, n, k; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - opgp = GA_Pgroup_get_default(); - - ierr = MPIComm2GApgroup(comm, &pgp); - GA_Pgroup_set_default(pgp); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Aga); CHKERRQ(ierr); - ierr = Mat2GA(A, Aga); CHKERRQ(ierr); - m = grows; - k = gcols; - - ierr = MatGetSize(B, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Bga); CHKERRQ(ierr); - ierr = Mat2GA(B, Bga); CHKERRQ(ierr); - n = gcols; - - ierr = MatGetSize(C, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(C, &lrows, &lcols); CHKERRQ(ierr); - ierr = CreateMatGA(pgp, lrows, lcols, grows, gcols, &Cga); CHKERRQ(ierr); - - char no('n'); - PetscScalarGA alpha, beta; - -#if defined(PETSC_USE_REAL_DOUBLE) -# if defined(PETSC_USE_COMPLEX) - - alpha.real = 1.0; - alpha.imag = 0.0; - beta.real = 0.0; - beta.imag = 0.0; - - GA_Zgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# else - - alpha = 1.0; - beta = 0.0; - - GA_Dgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# endif -#else -# if defined(PETSC_USE_COMPLEX) -# error "no single precision complex" -# else - - alpha = 1.0; - beta = 0.0; - - GA_Sgemm(no, no, m, n, k, one, Aga, Bga, beta, Cga); - -# endif -#endif - - ierr = GA2Mat(Cga, C); CHKERRQ(ierr); - - GA_Pgroup_sync(pgp); - - GA_Destroy(Aga); - GA_Destroy(Bga); - GA_Destroy(Cga); - - GA_Pgroup_set_default(opgp); - GA_Pgroup_destroy(pgp); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMultbyGA_new -// ------------------------------------------------------------- -PetscErrorCode -MatMultbyGA_new(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - MPI_Comm comm; - int nproc; - PetscInt grows_a, grows_b, grows_c; - PetscInt lrows_a, lrows_b, lrows_c; - PetscInt gcols_a, gcols_b, gcols_c; - PetscInt lcols_a, lcols_b, lcols_c; - - ierr = MatGetSize(A, &grows_a, &gcols_a); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows_a, &lcols_a); CHKERRQ(ierr); - - ierr = MatGetSize(B, &grows_b, &gcols_b); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &lrows_b, &lcols_b); CHKERRQ(ierr); - - grows_c = grows_a; - lrows_c = lrows_a; - gcols_c = gcols_b; - lcols_c = lcols_b; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - ierr = MPI_Comm_size(comm, &nproc); - - ierr = MatCreate(comm, &C); CHKERRQ(ierr); - ierr = MatSetSizes(C, lrows_c, lcols_c, grows_c, gcols_c); CHKERRQ(ierr); - if (nproc == 1) { - ierr = MatSetType(C, MATSEQDENSE); CHKERRQ(ierr); - ierr = MatSeqDenseSetPreallocation(C, PETSC_NULL); CHKERRQ(ierr); - } else { - ierr = MatSetType(C, MATDENSE); CHKERRQ(ierr); - ierr = MatMPIDenseSetPreallocation(C, PETSC_NULL); CHKERRQ(ierr); - } - - ierr = MatAssemblyBegin(C, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(C, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatMultbyGA(A, B, C); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// Vec2GA -// ------------------------------------------------------------- -static -PetscErrorCode -Vec2GA(Vec x, int pgroup, int *ga, bool trans = false) -{ - int lrows, rows; - PetscErrorCode ierr = 0; - - ierr = VecGetLocalSize(x, &lrows); CHKERRQ(ierr); - ierr = VecGetSize(x, &rows); CHKERRQ(ierr); - - PetscInt vlo, vhi; - ierr = VecGetOwnershipRange(x, &vlo, &vhi); CHKERRQ(ierr); - - const PetscScalar *v; - ierr = VecGetArrayRead(x, &v); CHKERRQ(ierr); - - int lo[2] = {0,0}, hi[2] = {0,0}, ld[2] = {1,1}; - if (!trans) { - ierr = CreateMatGA(pgroup, lrows, 1, rows, 1, ga); CHKERRQ(ierr); - lo[0] = vlo; - hi[0] = vhi-1; - } else { - ierr = CreateMatGA(pgroup, 1, lrows, 1, rows, ga); CHKERRQ(ierr); - lo[1] = vlo; - hi[1] = vhi-1; - } - // bad cast to get rid of const - NGA_Put(*ga, lo, hi, (void *)v, ld); - // GA_Print(*ga); - ierr = VecRestoreArrayRead(x, &v); CHKERRQ(ierr); - - GA_Pgroup_sync(pgroup); - return ierr; -} - -// ------------------------------------------------------------- -// GA2Vec -// ------------------------------------------------------------- -static -int -GA2Vec(int ga, Vec x) -{ - int lrows, rows; - PetscErrorCode ierr = 0; - - ierr = VecGetLocalSize(x, &lrows); CHKERRQ(ierr); - ierr = VecGetSize(x, &rows); CHKERRQ(ierr); - - PetscInt vlo, vhi; - ierr = VecGetOwnershipRange(x, &vlo, &vhi); CHKERRQ(ierr); - - PetscScalar *v; - ierr = VecGetArray(x, &v); CHKERRQ(ierr); - int lo[2] = {vlo,0}, hi[2] = {vhi-1,0}, ld[2] = {1,1}; - NGA_Get(ga, lo, hi, v, ld); - ierr = VecRestoreArray(x, &v); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatSetValues_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatSetValues_DenseGA(Mat mat, - PetscInt m, const PetscInt idxm[], PetscInt n, const PetscInt idxn[], - const PetscScalar v[],InsertMode addv) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int i, j, idx; - PetscScalar vij, one(1.0); - int lo[2], hi[2], ld[2] = {1, 1}; - ierr = MatShellGetContext(mat, (void *)&ctx); CHKERRQ(ierr); - - idx = 0; - for (i = 0; i < m; ++i) { - for (j = 0; j < n; ++j, ++idx) { - lo[0] = idxm[i]; - hi[0] = idxm[i]; - lo[1] = idxn[j]; - hi[1] = idxn[j]; - vij = v[idx]; - switch (addv) { - case INSERT_VALUES: - NGA_Put(ctx->ga, lo, hi, (void *)&vij, ld); - break; - case ADD_VALUES: - NGA_Acc(ctx->ga, lo, hi, (void *)&vij, ld, &one); - break; - default: - BOOST_ASSERT_MSG(false, "Unknown set operation"); - } - } - } - return ierr; -} - -// ------------------------------------------------------------- -// MatGetValues_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatGetValues_DenseGA(Mat mat, - PetscInt m, const PetscInt idxm[], PetscInt n, const PetscInt idxn[], - PetscScalar v[]) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int i, j, idx; - PetscScalar vij; - int lo[2], hi[2], ld[2] = {1, 1}; - ierr = MatShellGetContext(mat, (void *)&ctx); CHKERRQ(ierr); - - idx = 0; - for (i = 0; i < m; ++i) { - for (j = 0; j < n; ++j, ++idx) { - lo[0] = idxm[i]; - hi[0] = idxm[i]; - lo[1] = idxn[j]; - hi[1] = idxn[j]; - NGA_Get(ctx->ga, lo, hi, (void *)&vij, ld); - v[idx] = vij; - } - } - return ierr; -} - -// ------------------------------------------------------------- -// MatAssemmblyBegin_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatAssemmblyBegin_DenseGA(Mat mat, MatAssemblyType type) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// MatAssemmblyEnd_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatAssemmblyEnd_DenseGA(Mat mat, MatAssemblyType type) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - GA_Pgroup_sync(ctx->gaGroup); - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - // GA_Print(ctx->ga); - return ierr; -} - - -// ------------------------------------------------------------- -// MatMult_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatMult_DenseGA(Mat mat, Vec x, Vec y) -{ - // FIXME: I'm assuming the Mat and Vec's are compatible and that's - // been checked somewhere else. Probably a mistake. - - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - PetscInt Arows, Acols; - ierr = MatGetSize(mat, &Arows, &Acols); CHKERRQ(ierr); - - int g_x, g_y; - ierr = Vec2GA(x, ctx->gaGroup, &g_x, false); CHKERRQ(ierr); - ierr = Vec2GA(y, ctx->gaGroup, &g_y, false); CHKERRQ(ierr); - - PetscScalarGA alpha(one), beta(zero); - int ndim, itype, lo[2] = {0,0}, ahi[2], xhi[2], yhi[2]; - NGA_Inquire(ctx->ga, &itype, &ndim, ahi); - ahi[0] -= 1; ahi[1] -= 1; - NGA_Inquire(g_x, &itype, &ndim, xhi); - xhi[0] -= 1; xhi[1] -= 1; - NGA_Inquire(g_y, &itype, &ndim, yhi); - yhi[0] -= 1; yhi[1] -= 1; - // GA_Print(ctx->ga); - // GA_Print(g_x); - NGA_Matmul_patch('N', 'N', &alpha, &beta, - ctx->ga, lo, ahi, - g_x, lo, xhi, - g_y, lo, yhi); - - GA_Pgroup_sync(ctx->gaGroup); - // GA_Print(g_y); - - ierr = GA2Vec(g_y, y); CHKERRQ(ierr); - - GA_Destroy(g_y); - GA_Destroy(g_x); - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - - return ierr; -} - -// ------------------------------------------------------------- -// MatMatMult_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatMatMult_DenseGA(Mat A, Mat B, MatReuse scall, PetscReal fill, Mat *C) -{ - - // matrix sizes appear to be checked before here, so we won't do it again - - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - - MatType atype, btype; - ierr = MatGetType(A, &atype); - ierr = MatGetType(B, &btype); - - PetscBool issame; - PetscStrcmp(atype, btype, &issame); - - Mat Bga, Cga; - struct MatGACtx *Actx, *Bctx, *Cctx; - - ierr = MatShellGetContext(A, &Actx); CHKERRQ(ierr); - - if (issame) { - Bga = B; - } else { - ierr = MatConvertToDenseGA(B, &Bga); CHKERRQ(ierr); - } - ierr = MatShellGetContext(Bga, &Bctx); CHKERRQ(ierr); - - PetscInt lrows, lcols, grows, gcols, junk; - - ierr = MatGetSize(A, &grows, &junk); CHKERRQ(ierr); - ierr = MatGetSize(B, &junk, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &junk); CHKERRQ(ierr); - ierr = MatGetLocalSize(B, &junk, &lcols); CHKERRQ(ierr); - - ierr = MatCreateDenseGA(comm, lrows, lcols, grows, gcols, &Cga); CHKERRQ(ierr); - ierr = MatShellGetContext(Cga, &Cctx); CHKERRQ(ierr); - - PetscScalarGA alpha(one), beta(zero); - int ndim, itype, lo[2] = {0,0}, ahi[2], bhi[2], chi[2]; - NGA_Inquire(Actx->ga, &itype, &ndim, ahi); - ahi[0] -= 1; ahi[1] -= 1; - NGA_Inquire(Bctx->ga, &itype, &ndim, bhi); - bhi[0] -= 1; bhi[1] -= 1; - NGA_Inquire(Cctx->ga, &itype, &ndim, chi); - chi[0] -= 1; chi[1] -= 1; - // GA_Print(Actx->ga); - // GA_Print(Bctx->ga); - NGA_Matmul_patch('N', 'N', &alpha, &beta, - Actx->ga, lo, ahi, - Bctx->ga, lo, bhi, - Cctx->ga, lo, chi); - // GA_Print(Cctx->ga); - - switch (scall) { - case MAT_REUSE_MATRIX: - ierr = MatCopy(Cga, *C, SAME_NONZERO_PATTERN); CHKERRQ(ierr); - case MAT_INITIAL_MATRIX: - default: - ierr = MatDuplicate(Cga, MAT_COPY_VALUES, C); CHKERRQ(ierr); - break; - } - - if (!issame) ierr = MatDestroy(&Bga); CHKERRQ(ierr); - ierr = MatDestroy(&Cga); CHKERRQ(ierr); - - return ierr; -} - -// ------------------------------------------------------------- -// MatView_DenseGA -// ------------------------------------------------------------- - - -// ------------------------------------------------------------- -// MatZeroEntries_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatZeroEntries_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - - struct MatGACtx *ctx; - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - GA_Zero(ctx->ga); - - return ierr; -} - - -// ------------------------------------------------------------- -// MatDestroy_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatDestroy_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)A,&comm); CHKERRQ(ierr); - ierr = MPI_Barrier(comm); - - struct MatGACtx *ctx; - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - GA_Pgroup_sync(ctx->gaGroup); - GA_Destroy(ctx->ga); - GA_Pgroup_destroy(ctx->gaGroup); - ierr = PetscFree(ctx); - return ierr; -} - -// ------------------------------------------------------------- -// MatTranspose_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatTranspose_DenseGA(Mat mat, MatReuse reuse, Mat *B) -{ - PetscErrorCode ierr = 0; - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat, &comm); CHKERRQ(ierr); - - struct MatGACtx *ctx, *newctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - PetscInt lrows, grows, lcols, gcols; - ierr = MatGetSize(mat, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(mat, &lrows, &lcols); CHKERRQ(ierr); - - ierr = PetscMalloc(sizeof(struct MatGACtx), &newctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(newctx->gaGroup)); - - ierr = CreateMatGA(newctx->gaGroup, lcols, lrows, gcols, grows, &(newctx->ga)); CHKERRQ(ierr); - GA_Transpose(ctx->ga, newctx->ga); - - ierr = MatCreateShell(comm, lcols, lrows, gcols, grows, newctx, B); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*B); - - return ierr; -} - - -// ------------------------------------------------------------- -// MatCreateDenseGA -// ------------------------------------------------------------- -PetscErrorCode -MatCreateDenseGA(MPI_Comm comm, - PetscInt m,PetscInt n,PetscInt M,PetscInt N, - Mat *A) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx; - int lrows, grows, lcols, gcols; - - ierr = PetscMalloc(sizeof(struct MatGACtx), &ctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(ctx->gaGroup)); CHKERRQ(ierr); - - lrows = m; lcols = n; - grows = M; gcols = N; - - if (lrows == PETSC_DECIDE || lrows == PETSC_DETERMINE || - grows == PETSC_DECIDE || grows == PETSC_DETERMINE) { - ierr = PetscSplitOwnership(comm, &lrows, &grows); CHKERRQ(ierr); - } - - if (lcols == PETSC_DECIDE || lcols == PETSC_DETERMINE || - gcols == PETSC_DECIDE || gcols == PETSC_DETERMINE) { - ierr = PetscSplitOwnership(comm, &lcols, &gcols); CHKERRQ(ierr); - } - - ierr = CreateMatGA(ctx->gaGroup, lrows, lcols, grows, gcols, &(ctx->ga)); CHKERRQ(ierr); - ierr = MatCreateShell(comm, lrows, lcols, grows, gcols, ctx, A); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*A); - - return ierr; -} - -// ------------------------------------------------------------- -// MatDuplicate_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatDuplicate_DenseGA(Mat mat, MatDuplicateOption op, Mat *M) -{ - PetscErrorCode ierr = 0; - struct MatGACtx *ctx, *newctx; - ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr); - - MPI_Comm comm; - ierr = PetscObjectGetComm((PetscObject)mat, &comm); CHKERRQ(ierr); - - PetscInt lrows, grows, lcols, gcols; - ierr = MatGetSize(mat, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(mat, &lrows, &lcols); CHKERRQ(ierr); - - ierr = PetscMalloc(sizeof(struct MatGACtx), &newctx); CHKERRQ(ierr); - ierr = MPIComm2GApgroup(comm, &(newctx->gaGroup)); - - ierr = CreateMatGA(newctx->gaGroup, - lrows, lcols, grows, gcols, &(newctx->ga)); CHKERRQ(ierr); - ierr = MatCreateShell(comm, lrows, lcols, grows, gcols, newctx, M); CHKERRQ(ierr); - ierr = MatSetOperations_DenseGA(*M); - - PetscScalar z(0.0); - switch (op) { - case (MAT_COPY_VALUES): - GA_Copy(ctx->ga, newctx->ga); - break; - default: - GA_Fill(newctx->ga, &z); - break; - } - - GA_Pgroup_sync(newctx->gaGroup); - - return ierr; -} - -// ------------------------------------------------------------- -// MatConvertToDenseGA -// ------------------------------------------------------------- -PetscErrorCode -MatConvertToDenseGA(Mat A, Mat *B) -{ - PetscErrorCode ierr = 0; - MPI_Comm comm; - int lrows, grows, lcols, gcols; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - - ierr = MatCreateDenseGA(comm, lrows, lcols, grows, gcols, B); CHKERRQ(ierr); - ierr = MatCopy(A, *B, SAME_NONZERO_PATTERN); CHKERRQ(ierr); - - - return ierr; -} - -// ------------------------------------------------------------- -// MatConvertGAtoDense -// ------------------------------------------------------------- -PetscErrorCode -MatConvertGAToDense(Mat A, Mat *B) -{ - PetscErrorCode ierr = 0; - MPI_Comm comm; - int nproc; - struct MatGACtx *ctx; - PetscInt lrows, grows, lcols, gcols, lo, hi; - - ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr); - ierr = MPI_Comm_size(comm, &nproc); - - ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr); - - ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr); - - ierr = MatCreate(comm, B); CHKERRQ(ierr); - ierr = MatSetSizes(*B, lrows, lcols, grows, gcols); CHKERRQ(ierr); - if (nproc == 1) { - ierr = MatSetType(*B, MATSEQDENSE); CHKERRQ(ierr); - ierr = MatSeqDenseSetPreallocation(*B, PETSC_NULL); CHKERRQ(ierr); - } else { - ierr = MatSetType(*B, MATDENSE); CHKERRQ(ierr); - ierr = MatMPIDenseSetPreallocation(*B, PETSC_NULL); CHKERRQ(ierr); - } - ierr = MatGetOwnershipRange(*B, &lo, &hi); CHKERRQ(ierr); - - std::vector cidx(gcols); - for (PetscInt c = 0; c < gcols; ++c) { - cidx[c] = c; - } - std::vector rowvals(gcols); - for (PetscInt r = lo; r < hi; ++r) { - int glo[2] = {r, 0}; - int ghi[2] = {r, gcols - 1}; - int ld[2] = {1,1}; - NGA_Get(ctx->ga, glo, ghi, &rowvals[0], ld); - ierr = MatSetValues(*B, 1, &r, gcols, &cidx[0], &rowvals[0], INSERT_VALUES); CHKERRQ(ierr); - } - - ierr = MatAssemblyBegin(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - return ierr; -} - -// ------------------------------------------------------------- -// MatSetOperations_DenseGA -// ------------------------------------------------------------- -static -PetscErrorCode -MatSetOperations_DenseGA(Mat A) -{ - PetscErrorCode ierr = 0; - ierr = MatShellSetOperation(A, MATOP_SET_VALUES, (void(*)(void))MatSetValues_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_GET_VALUES, (void(*)(void))MatGetValues_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ZERO_ENTRIES, (void(*)(void))MatZeroEntries_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_MULT, (void(*)(void))MatMult_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_MAT_MULT, (void(*)(void))MatMatMult_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_TRANSPOSE, (void(*)(void))MatTranspose_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_DUPLICATE, (void(*)(void))MatDuplicate_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ASSEMBLY_BEGIN, (void(*)(void))MatAssemmblyBegin_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_ASSEMBLY_END, (void(*)(void))MatAssemmblyEnd_DenseGA); CHKERRQ(ierr); - ierr = MatShellSetOperation(A, MATOP_DESTROY, (void(*)(void))MatDestroy_DenseGA); CHKERRQ(ierr); - return ierr; -} - diff --git a/src/math/petsc/ga_matrix.hpp b/src/math/petsc/ga_matrix.hpp deleted file mode 100644 index 90292987c..000000000 --- a/src/math/petsc/ga_matrix.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* ------------------------------------------------------------- - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - * ------------------------------------------------------------- */ -/** - * @file ga_matrix.h - * @author William A. Perkins - * @date 2017-10-06 13:26:43 d3g096 - * - * @brief - * - * - */ - -#ifndef _ga_matrix_h_ -#define _ga_matrix_h_ - -#include "petscmat.h" - - -/** - * Create a dense PETSc matrix that uses Global Arrays for data - * storage. - * - * @param comm - * @param m local number of rows (or PETSC_DECIDE) - * @param n local number of columns (or PETSC_DECIDE) - * @param M total number of rows (or PETSC_DETERMINE) - * @param N total number of columns (or PETSC_DETERMINE) - * @param A new PETSc matrix instance - * - * @return - */ -extern -PetscErrorCode -MatCreateDenseGA(MPI_Comm comm, - PetscInt m,PetscInt n,PetscInt M,PetscInt N, - Mat *A); - - -/// Convert a PETSc Matrix to a GA-based one -/** - * - * @param A - * @param B - * - * @return - */ -extern -PetscErrorCode -MatConvertToDenseGA(Mat A, Mat *B); - -/// Convert a GA-based Matrix to a PETSc dense matrix -extern -PetscErrorCode -MatConvertGAToDense(Mat A, Mat *B); - - -extern -PetscErrorCode -MatMultbyGA(const Mat& A, const Mat& B, Mat& C); - -extern -PetscErrorCode -MatMultbyGA_new(const Mat& A, const Mat& B, Mat& C); - - -#endif diff --git a/src/math/petsc/petsc_matrix_operations.cpp b/src/math/petsc/petsc_matrix_operations.cpp index b89bb6909..4dc2d9ac9 100644 --- a/src/math/petsc/petsc_matrix_operations.cpp +++ b/src/math/petsc/petsc_matrix_operations.cpp @@ -8,7 +8,7 @@ /** * @file petsc_matrix_operations.cpp * @author William A. Perkins - * @date 2019-12-03 08:14:38 d3g096 + * @date 2023-08-23 13:14:17 d3g096 * * @brief * @@ -26,7 +26,6 @@ #include "petsc/petsc_matrix_extractor.hpp" #include "petsc/petsc_vector_implementation.hpp" #include "petsc/petsc_vector_extractor.hpp" -#include "petsc/ga_matrix.hpp" namespace gridpack { namespace math { @@ -35,6 +34,8 @@ namespace math { // ------------------------------------------------------------- // transpose +// This should be avoided in favor of MatrixT<> = transpose(MatrixT<>) +// because this can cause additional mallocs // ------------------------------------------------------------- template void @@ -47,10 +48,14 @@ transpose(const MatrixT& A, MatrixT& result) Mat *pAtrans(PETScMatrix(result)); PetscErrorCode ierr(0); try { + // + ierr = MatSetOption(*pAtrans, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE); CHKERRXX(ierr); ierr = MatTranspose(*pA, MAT_REUSE_MATRIX, pAtrans); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } + // This appears to be necessary with PETSc >= 3.12 + result.ready(); if (!PETScMatrixImplementation::useLibrary) { result.conjugate(); } @@ -129,6 +134,7 @@ transpose(const MatrixT& A) } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } + result->ready(); if (!PETScMatrixImplementation::useLibrary) { result->conjugate(); } @@ -357,27 +363,6 @@ multiply(const MatrixT& A, VectorT& result); -// ------------------------------------------------------------- -// multiply_dense -// ------------------------------------------------------------- -static -PetscErrorCode -multiply_dense(const Mat& A, const Mat& B, Mat& C) -{ - PetscErrorCode ierr(0); - Mat Aga, Bga, Cga; - - ierr = MatConvertToDenseGA(A, &Aga); CHKERRQ(ierr); - ierr = MatConvertToDenseGA(B, &Bga); CHKERRQ(ierr); - ierr = MatMatMult(Aga, Bga, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Cga); CHKERRQ(ierr); - ierr = MatConvertGAToDense(Cga, &C); CHKERRXX(ierr); - - ierr = MatDestroy(&Aga); CHKERRQ(ierr); - ierr = MatDestroy(&Bga); CHKERRQ(ierr); - ierr = MatDestroy(&Cga); CHKERRQ(ierr); - return ierr; -} - // ------------------------------------------------------------- // (Matrix) multiply // ------------------------------------------------------------- @@ -387,16 +372,6 @@ multiply(const MatrixT& A, const MatrixT& B, MatrixT& result) { PetscErrorCode ierr(0); - // special method required for parallel dense*dense - if (A.communicator().size() > 1 && - A.storageType() == Dense && - B.storageType() == Dense) { - const Mat *Amat(PETScMatrix(A)); - const Mat *Bmat(PETScMatrix(B)); - Mat *Cmat(PETScMatrix(result)); - ierr = MatMultbyGA(*Amat, *Bmat, *Cmat); CHKERRXX(ierr); - // ierr = multiply_dense(*Amat, *Bmat, *Cmat); CHKERRXX(ierr); - } else { const Mat *Amat(PETScMatrix(A)); const Mat *Bmat(PETScMatrix(B)); Mat *Cmat(PETScMatrix(result)); @@ -407,7 +382,7 @@ multiply(const MatrixT& A, const MatrixT& B, MatrixT& result) } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); } - } + } template @@ -428,19 +403,6 @@ multiply(const MatrixT& A, const MatrixT& B) { PetscErrorCode ierr(0); MatrixT *result; - // special method required for parallel dense*dense - if (A.communicator().size() > 1 && - A.storageType() == Dense && - B.storageType() == Dense) { - const Mat *Amat(PETScMatrix(A)); - const Mat *Bmat(PETScMatrix(B)); - Mat Cmat; - ierr = MatMultbyGA_new(*Amat, *Bmat, Cmat); - - PETScMatrixImplementation *result_impl = - new PETScMatrixImplementation(Cmat, false, true); - result = new MatrixT(result_impl); - } else { const Mat *Amat(PETScMatrix(A)); const Mat *Bmat(PETScMatrix(B)); Mat Cmat; @@ -454,7 +416,7 @@ multiply(const MatrixT& A, const MatrixT& B) PETScMatrixImplementation *result_impl = new PETScMatrixImplementation(Cmat, false, true); result = new MatrixT(result_impl); - } + return result; } diff --git a/src/math/test/petsc_ga_matrix.cpp b/src/math/test/petsc_ga_matrix.cpp deleted file mode 100644 index f815b1727..000000000 --- a/src/math/test/petsc_ga_matrix.cpp +++ /dev/null @@ -1,358 +0,0 @@ -// ------------------------------------------------------------- -/* - * Copyright (c) 2013 Battelle Memorial Institute - * Licensed under modified BSD License. A copy of this license can be found - * in the LICENSE file in the top level directory of this distribution. - */ - -// ------------------------------------------------------------- -// ------------------------------------------------------------- -/** - * @file petsc_ga_matrix.cpp - * @author William A. Perkins - * @date 2019-08-01 08:51:08 d3g096 - * - * @brief - * - * - */ -// ------------------------------------------------------------- - -#include - -#include "math.hpp" -#include "gridpack/parallel/parallel.hpp" -#include "petsc/ga_matrix.hpp" -#include "petsc/petsc_exception.hpp" - -#include "test_main.cpp" - -static const PetscInt local_size(5); - -static -PetscErrorCode -fill_pattern(Mat A, InsertMode addv) -{ - PetscErrorCode ierr(0); - PetscScalar x(0.0); - PetscInt lo, hi; - - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRQ(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatSetValues(A, 1, &i, 1, &j, &x, addv); CHKERRQ(ierr); - x += 1.0; - } - } - - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); - return ierr; -} - -static -void -convert_and_check(Mat A) -{ - PetscErrorCode ierr(0); - PetscInt lo, hi; - Mat B; - - ierr = MatConvertToDenseGA(A, &B); CHKERRXX(ierr); - ierr = MatGetOwnershipRange(B, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - PetscScalar x; - PetscScalar y; - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(B, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&B); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_SUITE(GAMatrixTest) - -BOOST_AUTO_TEST_CASE( ConstructConvertDuplicate ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A, B, C; - PetscScalar x(0.0); - PetscInt lrows, lcols; - PetscInt lo, hi; - - - BOOST_TEST_MESSAGE("Create Matrix"); - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, 5); - BOOST_CHECK_EQUAL(lcols, 5); - - BOOST_TEST_MESSAGE("Fill Matrix"); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - PetscScalar y(0.0); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - y += 1.0; - } - } - - BOOST_TEST_MESSAGE("Add Matrix Values"); - - ierr = fill_pattern(A, ADD_VALUES); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - y = 0.0; - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, 2.0*y); - y += 1.0; - } - } - - BOOST_TEST_MESSAGE("Convert Matrix"); - - ierr = MatConvert(A, MATDENSE, MAT_INITIAL_MATRIX, &B); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("View Converted Matrix"); - - ierr = MatView(B, PETSC_VIEWER_STDOUT_WORLD); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(B, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&B); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Duplicate Matrix"); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &C); CHKERRXX(ierr); - - BOOST_TEST_MESSAGE("Get Matrix Values"); - - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - ierr = MatGetValues(A, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(C, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - ierr = MatDestroy(&C); CHKERRXX(ierr); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( Dense2GA ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - Mat A; - PetscInt lrows = 5, lcols = 5; - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, lrows, lcols, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATDENSE); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - convert_and_check(A); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( Sparse2GA ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - Mat A; - PetscInt lrows = 5, lcols = 5; - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, lrows, lcols, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATAIJ); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - - convert_and_check(A); - - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - - -BOOST_AUTO_TEST_CASE( VectorMultiply ) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A; - PetscInt lrows, lcols; - PetscInt grows, gcols; - PetscInt lo, hi; - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - ierr = MatGetSize(A, &grows, &gcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, local_size); - BOOST_CHECK_EQUAL(lcols, local_size); - - PetscScalar v(2.0); - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - ierr = MatSetValues(A, 1, &i, 1, &i, &v, INSERT_VALUES); CHKERRXX(ierr); - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - - Vec x, y; - ierr = VecCreate(world, &x); CHKERRXX(ierr); - ierr = VecSetSizes(x, lcols, gcols); CHKERRXX(ierr); - ierr = VecSetFromOptions(x); CHKERRXX(ierr); - v = 1.0; - ierr = VecSet(x, v); CHKERRXX(ierr); - ierr = VecAssemblyBegin(x); - ierr = VecAssemblyEnd(x); - - ierr = VecCreate(world, &y); CHKERRXX(ierr); - ierr = VecSetSizes(y, lrows, grows); CHKERRXX(ierr); - ierr = VecSetFromOptions(y); CHKERRXX(ierr); - v = 0.0; - ierr = VecSet(y, v); CHKERRXX(ierr); - ierr = VecAssemblyBegin(y); - ierr = VecAssemblyEnd(y); - - ierr = MatMult(A, x, y); CHKERRXX(ierr); - - ierr = VecGetOwnershipRange(y, &lo, &hi); CHKERRXX(ierr); - v = 2.0; - for (int i = lo; i < hi; ++i) { - PetscScalar xtmp; - ierr = VecGetValues(y, 1, &i, &xtmp); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(v, xtmp); - } - - ierr = VecDestroy(&x); CHKERRXX(ierr); - ierr = VecDestroy(&y); CHKERRXX(ierr); - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_CASE( SquareMatrixMultiply ) -{ - Mat A, B, C, Cbase; - PetscErrorCode ierr(0); - gridpack::parallel::Communicator comm; - - // do a multiply with a pair of normal sparse matrices for a check - // (dense multiply may not be available in parallel) - - ierr = MatCreate(comm, &A); CHKERRXX(ierr); - ierr = MatSetSizes(A, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE); CHKERRXX(ierr); - ierr = MatSetType(A, MATAIJ); CHKERRXX(ierr); - ierr = MatSetFromOptions(A); CHKERRXX(ierr); - ierr = MatSetUp(A); CHKERRXX(ierr); - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &B); CHKERRXX(ierr); - ierr = MatMatMult(A, B, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Cbase); CHKERRXX(ierr); - - ierr = MatDestroy(&A); CHKERRXX(ierr); - ierr = MatDestroy(&B); CHKERRXX(ierr); - - // Do the same multiply with GA-based matrices - - ierr = MatCreateDenseGA(comm, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); - CHKERRXX(ierr); - ierr = fill_pattern(A, INSERT_VALUES); CHKERRXX(ierr); - ierr = MatDuplicate(A, MAT_COPY_VALUES, &B); CHKERRXX(ierr); - - ierr = MatMatMult(A, B, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &C); - PetscInt lo, hi; - ierr = MatGetOwnershipRange(Cbase, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - PetscScalar x; - PetscScalar y; - ierr = MatGetValues(Cbase, 1, &i, 1, &j, &x); CHKERRXX(ierr); - ierr = MatGetValues(C, 1, &i, 1, &j, &y); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(x, y); - } - } - - ierr = MatDestroy(&A); CHKERRXX(ierr); - ierr = MatDestroy(&B); CHKERRXX(ierr); - ierr = MatDestroy(&C); CHKERRXX(ierr); - ierr = MatDestroy(&Cbase); CHKERRXX(ierr); - -} - -BOOST_AUTO_TEST_CASE(Transpose) -{ - PetscErrorCode ierr(0); - gridpack::parallel::Communicator world; - - Mat A, Atrans; - PetscInt lrows, lcols; - PetscInt grows, gcols; - PetscInt lo, hi; - - ierr = MatCreateDenseGA(world, local_size, local_size, PETSC_DETERMINE, PETSC_DETERMINE, &A); CHKERRXX(ierr); - ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRXX(ierr); - ierr = MatGetSize(A, &grows, &gcols); CHKERRXX(ierr); - - BOOST_CHECK_EQUAL(lrows, local_size); - BOOST_CHECK_EQUAL(lcols, local_size); - - PetscScalar v; - ierr = MatGetOwnershipRange(A, &lo, &hi); CHKERRXX(ierr); - for (int i = lo; i < hi; ++i) { - v = (PetscScalar)i; - for (int j = lo; j < hi; ++j) { - ierr = MatSetValues(A, 1, &i, 1, &j, &v, INSERT_VALUES); CHKERRXX(ierr); - } - } - ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRXX(ierr); - - ierr = MatTranspose(A, MAT_INITIAL_MATRIX, &Atrans); CHKERRXX(ierr); - - PetscScalar x; - for (int i = lo; i < hi; ++i) { - for (int j = lo; j < hi; ++j) { - v = (PetscScalar)j; - ierr = MatGetValues(Atrans, 1, &i, 1, &j, &x); CHKERRXX(ierr); - BOOST_CHECK_EQUAL(v, x); - } - } - - ierr = MatDestroy(&Atrans); CHKERRXX(ierr); - ierr = MatDestroy(&A); CHKERRXX(ierr); -} - -BOOST_AUTO_TEST_SUITE_END() - - - - From e85428a1db99fe043d001408a769f76086270fdd Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 24 Aug 2023 10:41:33 -0700 Subject: [PATCH 013/125] Remove all PETSC_VERSION usage. --- src/math/petsc/petsc_configurable.cpp | 6 +- .../petsc/petsc_dae_solver_implementation.hpp | 59 +------------------ src/math/petsc/petsc_exception.hpp | 19 +----- .../petsc/petsc_linear_matrix_solver_impl.hpp | 12 +--- .../petsc_linear_solver_implementation.hpp | 6 +- .../petsc/petsc_matrix_implementation.hpp | 27 +-------- src/math/petsc/petsc_matrix_operations.cpp | 8 +-- src/math/petsc/petsc_matrix_wrapper.cpp | 26 +------- .../petsc_nonlinear_solver_implementation.hpp | 36 +---------- src/math/petsc/petsc_vector_wrapper.cpp | 9 +-- src/math/vector_implementation.hpp | 3 +- 11 files changed, 14 insertions(+), 197 deletions(-) diff --git a/src/math/petsc/petsc_configurable.cpp b/src/math/petsc/petsc_configurable.cpp index 23d2879e6..d9389df71 100644 --- a/src/math/petsc/petsc_configurable.cpp +++ b/src/math/petsc/petsc_configurable.cpp @@ -9,7 +9,7 @@ /** * @file petsc_configurable.cpp * @author William A. Perkins - * @date 2016-06-16 12:24:02 d3g096 + * @date 2023-08-24 09:41:08 d3g096 * * @brief * @@ -173,9 +173,7 @@ PETScConfigurable::p_processOptions(utility::Configuration::CursorPtr props) PetscErrorCode ierr(0); try { ierr = PetscOptionsInsertString( -#if PETSC_VERSION_GE(3,7,0) NULL, -#endif p_loadedOptions.c_str()); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -202,9 +200,7 @@ PETScConfigurable::p_unprocessOptions(void) << *o << "\"" << std::endl; } ierr = PetscOptionsClearValue( -#if PETSC_VERSION_GE(3,7,0) NULL, -#endif o->c_str()); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); diff --git a/src/math/petsc/petsc_dae_solver_implementation.hpp b/src/math/petsc/petsc_dae_solver_implementation.hpp index f3fa65482..f10886fda 100644 --- a/src/math/petsc/petsc_dae_solver_implementation.hpp +++ b/src/math/petsc/petsc_dae_solver_implementation.hpp @@ -10,7 +10,7 @@ /** * @file petsc_dae_solver_implementation.hpp * @author William A. Perkins - * @date 2019-12-05 10:01:29 d3g096 + * @date 2023-08-24 09:42:12 d3g096 * * @brief * @@ -156,15 +156,9 @@ class PETScDAESolverImplementation pterm[i] = (gterm[i] ? PETSC_TRUE : PETSC_FALSE); } -#if PETSC_VERSION_LT(3,7,0) - ierr = TSSetEventMonitor(p_ts, ne, &(pdir[0]), &(pterm[0]), - &EventHandler, &PostEventHandler, - NULL); -#else ierr = TSSetEventHandler(p_ts, ne, &(pdir[0]), &(pterm[0]), &EventHandler, &PostEventHandler, NULL); -#endif } ierr = TSSetProblemType(p_ts, TS_NONLINEAR); CHKERRXX(ierr); @@ -201,12 +195,8 @@ class PETScDAESolverImplementation { PetscErrorCode ierr(0); try { -#if PETSC_VERSION_LT(3,8,0) - ierr = TSSetInitialTimeStep(p_ts, t0, deltat0); CHKERRXX(ierr); -#else ierr = TSSetTime(p_ts, t0); CHKERRXX(ierr); ierr = TSSetTimeStep(p_ts, deltat0); CHKERRXX(ierr); -#endif Vec *xvec(PETScVector(x0)); ierr = TSSetSolution(p_ts, *xvec); } catch (const PETSC_EXCEPTION_TYPE& e) { @@ -221,13 +211,8 @@ class PETScDAESolverImplementation { PetscErrorCode ierr(0); try { -#if PETSC_VERSION_LT(3,8,0) - ierr = TSSetDuration(p_ts, maxsteps, maxtime); CHKERRXX(ierr); -#else ierr = TSSetMaxSteps(p_ts, maxsteps); CHKERRXX(ierr); ierr = TSSetMaxTime(p_ts, maxtime); CHKERRXX(ierr); -#endif - ierr = TSSetExactFinalTime(p_ts, TS_EXACTFINALTIME_MATCHSTEP); CHKERRXX(ierr); ierr = TSSolve(p_ts, PETSC_NULL); // std::cout << this->processor_rank() << ": " @@ -239,11 +224,7 @@ class PETScDAESolverImplementation ierr = TSGetConvergedReason(p_ts, &reason); CHKERRXX(ierr); PetscInt nstep; -#if PETSC_VERSION_LT(3,8,0) - ierr = TSGetTimeStepNumber(p_ts,&nstep);CHKERRXX(ierr); -#else ierr = TSGetStepNumber(p_ts,&nstep);CHKERRXX(ierr); -#endif maxsteps = nstep; if (reason >= 0) { @@ -285,42 +266,6 @@ class PETScDAESolverImplementation } } - -#if PETSC_VERSION_LT(3,5,0) - - /// Routine to assemble Jacobian that is sent to PETSc - static PetscErrorCode FormIJacobian(TS ts, PetscReal t, Vec x, Vec xdot, - PetscReal a, Mat *jac, Mat *B, - MatStructure *flag, void *dummy) - { - PetscErrorCode ierr(0); - - // Necessary C cast - PETScDAESolverImplementation *solver = - (PETScDAESolverImplementation *)dummy; - - // Copy PETSc's current estimate into - - // Should be the case, but just make sure - BOOST_ASSERT(*jac == *solver->p_petsc_J); - BOOST_ASSERT(*B == *solver->p_petsc_J); - - boost::scoped_ptr - xtmp(new VectorType(new PETScVectorImplementation(x, false))), - xdottmp(new VectorType(new PETScVectorImplementation(xdot, false))); - - // Call the user-specified function (object) to form the Jacobian - (solver->p_Jbuilder)(t, *xtmp, *xdottmp, a, *(solver->p_J)); - - *flag = SAME_NONZERO_PATTERN; - - return ierr; - - } - - -#else - /// Routine to assemble Jacobian that is sent to PETSc static PetscErrorCode FormIJacobian(TS ts, PetscReal t, Vec x, Vec xdot, PetscReal a, Mat jac, Mat B, @@ -350,8 +295,6 @@ class PETScDAESolverImplementation } -#endif - /// Routine to assemble RHS that is sent to PETSc static PetscErrorCode FormIFunction(TS ts, PetscReal t, Vec x, Vec xdot, Vec F, void *dummy) diff --git a/src/math/petsc/petsc_exception.hpp b/src/math/petsc/petsc_exception.hpp index a11a492da..edb6313d6 100644 --- a/src/math/petsc/petsc_exception.hpp +++ b/src/math/petsc/petsc_exception.hpp @@ -8,7 +8,7 @@ /** * @file petsc_exception.hpp * @author William A. Perkins - * @date 2019-08-13 09:12:24 d3g096 + * @date 2023-08-24 09:43:26 d3g096 * * @brief * @@ -26,29 +26,12 @@ // You gotta love PETSc consistency -#if PETSC_VERSION_(3,4,0) -#undef PETSC_VERSION_RELEASE -#define PETSC_VERSION_RELEASE 0 -#endif - // Default type for PETSc exceptions #define PETSC_EXCEPTION_TYPE std::runtime_error // If PETSc is compiled with the C++, use its C++ exception // facility. Otherwise, we need to make our own. -#ifdef PETSC_CLANGUAGE_CXX -// With PETSc version 3.5, PETSc::Exception was no longer defined. It -// was replaced with std::runtime_error. - -#if PETSC_VERSION_LT(3,5,0) -#include -#undef PETSC_EXCEPTION_TYPE -#define PETSC_EXCEPTION_TYPE PETSc::Exception -#define GRIDPACK_USES_PETSC_EXCEPTION 1 -#endif -#endif - #include "gridpack/utilities/exception.hpp" namespace gridpack { diff --git a/src/math/petsc/petsc_linear_matrix_solver_impl.hpp b/src/math/petsc/petsc_linear_matrix_solver_impl.hpp index 4c419af3a..ae577178c 100644 --- a/src/math/petsc/petsc_linear_matrix_solver_impl.hpp +++ b/src/math/petsc/petsc_linear_matrix_solver_impl.hpp @@ -10,7 +10,7 @@ /** * @file petsc_linear_matrx_solver_impl.hpp * @author William A. Perkins - * @date 2019-12-03 08:18:32 d3g096 + * @date 2023-08-24 09:43:53 d3g096 * * @brief * @@ -84,11 +84,7 @@ class PetscLinearMatrixSolverImplementation /// Choose a matrix solver type based on PETSc version typedef -#if PETSC_VERSION_LT(3,9,0) - MatSolverPackage -#else MatSolverType -#endif ThePetscMatSolverType; /// The underlying PETSc factored coefficient matrix @@ -292,12 +288,6 @@ PetscLinearMatrixSolverImplementation::p_supportedOrderingType[] = { MATORDERINGRCM, MATORDERINGQMD, MATORDERINGROWLENGTH -#if PETSC_VERSION_GT(3,5,0) - , - MATORDERINGWBM, - MATORDERINGSPECTRAL, - MATORDERINGAMD -#endif }; template diff --git a/src/math/petsc/petsc_linear_solver_implementation.hpp b/src/math/petsc/petsc_linear_solver_implementation.hpp index 0309dc9e1..33569c404 100644 --- a/src/math/petsc/petsc_linear_solver_implementation.hpp +++ b/src/math/petsc/petsc_linear_solver_implementation.hpp @@ -9,7 +9,7 @@ /** * @file petsc_linear_solver_implementation.hpp * @author William A. Perkins - * @date 2015-08-18 13:40:01 d3g096 + * @date 2023-08-24 09:44:02 d3g096 * * @brief * @@ -121,11 +121,7 @@ class PETScLinearSolverImplementation if (p_matrixSet && this->p_constSerialMatrix) { // KSPSetOperators can be skipped } else { -#if PETSC_VERSION_LT(3,5,0) - ierr = KSPSetOperators(p_KSP, *Amat, *Amat, SAME_NONZERO_PATTERN); CHKERRXX(ierr); -#else ierr = KSPSetOperators(p_KSP, *Amat, *Amat); CHKERRXX(ierr); -#endif p_matrixSet = true; } diff --git a/src/math/petsc/petsc_matrix_implementation.hpp b/src/math/petsc/petsc_matrix_implementation.hpp index e83515af2..91314e809 100644 --- a/src/math/petsc/petsc_matrix_implementation.hpp +++ b/src/math/petsc/petsc_matrix_implementation.hpp @@ -9,7 +9,7 @@ /** * @file petsc_matrix_implementation.h * @author William A. Perkins - * @date 2019-12-03 08:14:08 d3g096 + * @date 2023-08-24 09:44:50 d3g096 * * @brief * @@ -501,16 +501,10 @@ class PETScMatrixImplementation ridx[i] = rows[i]*elementSize; } -#if PETSC_VERSION_LT(3,6,0) - std::vector permidx(sortPermutation(ridx)); - ierr = ISCreateGeneral(PETSC_COMM_SELF, nrow, &permidx[0], - PETSC_COPY_VALUES, &irowperm); CHKERRXX(ierr); -#else ierr = ISCreateGeneral(PETSC_COMM_SELF, nrow, &ridx[0], PETSC_COPY_VALUES, &irow); CHKERRXX(ierr); ierr = ISSortPermutation(irow, PETSC_TRUE, &irowperm); CHKERRXX(ierr); ierr = ISDestroy(&irow); CHKERRXX(ierr); -#endif ierr = ISSetPermutation(irowperm); CHKERRXX(ierr); ierr = ISInvertPermutation(irowperm, PETSC_DECIDE, &irowinvert); CHKERRXX(ierr); ierr = ISDestroy(&irowperm); CHKERRXX(ierr); @@ -534,12 +528,8 @@ class PETScMatrixImplementation // ierr = PetscViewerDestroy(&v); CHKERRXX(ierr); Mat *sub; -#if PETSC_VERSION_LT(3,8,0) - ierr = MatGetSubMatrices(*mat, 1, &irow, &icol, MAT_INITIAL_MATRIX, &sub); CHKERRXX(ierr); -#else ierr = MatCreateSubMatrices(*mat, 1, &irow, &icol, MAT_INITIAL_MATRIX, &sub); CHKERRXX(ierr); - -#endif + std::vector cval(ncol); for (PetscInt j = 0; j < ncol; ++j) { cidx[j] = j; @@ -570,11 +560,7 @@ class PETScMatrixImplementation ierr = ISDestroy(&irow); CHKERRXX(ierr); ierr = ISDestroy(&icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatDestroyMatrices(1, &sub); CHKERRXX(ierr); -#else ierr = MatDestroySubMatrices(1, &sub); CHKERRXX(ierr); -#endif } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -700,11 +686,7 @@ class PETScMatrixImplementation ierr = ISCreateStride(this->communicator(), p_mwrap->rows(), 0, 1, &irow); CHKERRXX(ierr); ierr = ISCreateStride(this->communicator(), p_mwrap->cols(), 0, 1, &icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatGetSubMatrices(*Aorig, 1, &irow, &icol, MAT_INITIAL_MATRIX, &Aloc); CHKERRXX(ierr); -#else ierr = MatCreateSubMatrices(*Aorig, 1, &irow, &icol, MAT_INITIAL_MATRIX, &Aloc); CHKERRXX(ierr); -#endif // MatType mt; // ierr = MatGetType(*Aloc, &mt); CHKERRXX(ierr); @@ -716,12 +698,7 @@ class PETScMatrixImplementation ierr = ISDestroy(&irow); CHKERRXX(ierr); ierr = ISDestroy(&icol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,8,0) - ierr = MatDestroyMatrices(1, &Aloc); CHKERRXX(ierr); -#else ierr = MatDestroySubMatrices(1, &Aloc); CHKERRXX(ierr); -#endif - return result; } diff --git a/src/math/petsc/petsc_matrix_operations.cpp b/src/math/petsc/petsc_matrix_operations.cpp index 4dc2d9ac9..52367142a 100644 --- a/src/math/petsc/petsc_matrix_operations.cpp +++ b/src/math/petsc/petsc_matrix_operations.cpp @@ -8,7 +8,7 @@ /** * @file petsc_matrix_operations.cpp * @author William A. Perkins - * @date 2023-08-23 13:14:17 d3g096 + * @date 2023-08-24 09:45:38 d3g096 * * @brief * @@ -559,16 +559,10 @@ matrixLoadBinary(const parallel::Communicator& comm, const char* filename) filename, FILE_MODE_READ, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); CHKERRXX(ierr); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); CHKERRXX(ierr); -#endif ierr = MatLoad(mat, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); PETScMatrixImplementation *result_impl = diff --git a/src/math/petsc/petsc_matrix_wrapper.cpp b/src/math/petsc/petsc_matrix_wrapper.cpp index 82e1fffe3..d87fbee45 100644 --- a/src/math/petsc/petsc_matrix_wrapper.cpp +++ b/src/math/petsc/petsc_matrix_wrapper.cpp @@ -9,7 +9,7 @@ /** * @file petsc_matrix_wrapper.cpp * @author William A. Perkins - * @date 2019-09-13 12:53:47 d3g096 + * @date 2023-08-24 09:46:49 d3g096 * * @brief * @@ -472,22 +472,14 @@ petsc_print_matrix(const Mat mat, const char* filename, PetscViewerFormat format } else { ierr = PetscViewerASCIIGetStdout(comm, &viewer); CHKERRXX(ierr); } -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, format); -#else - ierr = PetscViewerSetFormat(viewer, format); -#endif CHKERRXX(ierr); PetscInt grow, gcol, lrow, lcol; switch (format) { case PETSC_VIEWER_DEFAULT: ierr = MatGetSize(mat, &grow, &gcol); CHKERRXX(ierr); ierr = MatGetLocalSize(mat, &lrow, &lcol); CHKERRXX(ierr); -#if PETSC_VERSION_LT(3,7,0) - ierr = PetscViewerASCIISynchronizedAllow(viewer, PETSC_TRUE); CHKERRXX(ierr); -#else ierr = PetscViewerASCIIPushSynchronized(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerASCIIPrintf(viewer, "# Matrix distribution\n"); CHKERRXX(ierr); ierr = PetscViewerASCIIPrintf(viewer, @@ -502,16 +494,12 @@ petsc_print_matrix(const Mat mat, const char* filename, PetscViewerFormat format "# ---- -------- --------\n"); CHKERRXX(ierr); ierr = PetscViewerASCIIPrintf(viewer, "# %4d %8d %8d\n", nproc, grow, gcol); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerASCIIPopSynchronized(viewer); CHKERRXX(ierr); -#endif default: break; } ierr = MatView(mat, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif if (filename != NULL) { ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); @@ -557,16 +545,10 @@ PetscMatrixWrapper::loadBinary(const char* filename) filename, FILE_MODE_READ, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); -#endif CHKERRXX(ierr); ierr = MatLoad(p_matrix, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); @@ -587,16 +569,10 @@ PetscMatrixWrapper::saveBinary(const char* filename) const filename, FILE_MODE_WRITE, &viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE); -#else - ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_NATIVE); -#endif CHKERRXX(ierr); ierr = MatView(p_matrix, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } catch (const PETSC_EXCEPTION_TYPE& e) { throw PETScException(ierr, e); diff --git a/src/math/petsc/petsc_nonlinear_solver_implementation.hpp b/src/math/petsc/petsc_nonlinear_solver_implementation.hpp index 375ba0c51..f7eb3b240 100644 --- a/src/math/petsc/petsc_nonlinear_solver_implementation.hpp +++ b/src/math/petsc/petsc_nonlinear_solver_implementation.hpp @@ -8,7 +8,7 @@ /** * @file petsc_nonlinear_solver_implementation.hpp * @author William A. Perkins - * @date 2015-03-26 14:25:10 d3g096 + * @date 2023-08-24 09:47:08 d3g096 * * @brief * @@ -188,38 +188,6 @@ class PetscNonlinearSolverImplementation } -#if PETSC_VERSION_LT(3,5,0) - /// Routine to assemble Jacobian that is sent to PETSc - static PetscErrorCode FormJacobian(SNES snes, Vec x, Mat *jac, Mat *B, - MatStructure *flag, void *dummy) - { - PetscErrorCode ierr(0); - - // Necessary C cast - PetscNonlinearSolverImplementation *solver = - (PetscNonlinearSolverImplementation *)dummy; - - // Copy PETSc's current estimate into - - // Should be the case, but just make sure - BOOST_ASSERT(*jac == *solver->p_petsc_J); - BOOST_ASSERT(*B == *solver->p_petsc_J); - - // Not sure about this - BOOST_ASSERT(x == *(solver->p_petsc_X)); - - // May need to do this, which seems slow. - // ierr = VecCopy(x, *(solver->p_petsc_X)); CHKERRQ(ierr); - - // Call the user-specified function (object) to form the Jacobian - (solver->p_jacobian)(*(solver->p_X), *(solver->p_J)); - - *flag = SAME_NONZERO_PATTERN; - - return ierr; - } - -#else /// Routine to assemble Jacobian that is sent to PETSc static PetscErrorCode FormJacobian(SNES snes, Vec x, Mat jac, Mat B, void *dummy) @@ -248,8 +216,6 @@ class PetscNonlinearSolverImplementation return ierr; } -#endif - /// Routine to assemble RHS that is sent to PETSc static PetscErrorCode FormFunction(SNES snes, Vec x, Vec f, void *dummy) { diff --git a/src/math/petsc/petsc_vector_wrapper.cpp b/src/math/petsc/petsc_vector_wrapper.cpp index 8c77fa39a..15e5bd893 100644 --- a/src/math/petsc/petsc_vector_wrapper.cpp +++ b/src/math/petsc/petsc_vector_wrapper.cpp @@ -8,7 +8,7 @@ /** * @file petsc_vector_implementation.cpp * @author William A. Perkins - * @date 2019-09-13 13:56:23 d3g096 + * @date 2023-08-24 09:47:21 d3g096 * * @brief * @@ -362,16 +362,11 @@ petsc_print_vector(const Vec vec, const char* filename, PetscViewerFormat format } else { ierr = PetscViewerASCIIGetStdout(comm, &viewer); CHKERRXX(ierr); } -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPushFormat(viewer, format); -#else - ierr = PetscViewerSetFormat(viewer, format); -#endif CHKERRXX(ierr); ierr = VecView(vec, viewer); CHKERRXX(ierr); -#if PETSC_VERSION_GE(3,7,0) ierr = PetscViewerPopFormat(viewer); CHKERRXX(ierr); -#endif + if (filename != NULL) { ierr = PetscViewerDestroy(&viewer); CHKERRXX(ierr); } diff --git a/src/math/vector_implementation.hpp b/src/math/vector_implementation.hpp index 06aae47b1..bf2553cc8 100644 --- a/src/math/vector_implementation.hpp +++ b/src/math/vector_implementation.hpp @@ -9,7 +9,7 @@ /** * @file vector_implementation.h * @author William A. Perkins - * @date 2019-11-20 10:01:00 d3g096 + * @date 2023-08-23 14:53:57 d3g096 * * @brief * @@ -162,6 +162,7 @@ class VectorImplementation IdxType lo, hi; this->localIndexRange(lo, hi); this->setElementRange(lo, hi, p_localElements.get()); + this->ready(); // may want to keep this around if operation is done alot p_localElements.reset(); } From b9f6171b77167d67d9ec2b3526dd87adcb099245 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 11:50:28 -0700 Subject: [PATCH 014/125] Update PETSc settings for tlaloc --- src/example_configuration.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index ee870d678..afe9e6ffb 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -256,13 +256,13 @@ elif [ $host == "tlaloc" ]; then export CC CXX CFLAGS CXXFLAGS # Custom built 3.16, complex: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - # -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ # Custom built 3.16, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.16.6" \ + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ # Custom built 3.14, complex: @@ -270,14 +270,19 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ + # Custom built 3.14, real: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ + # -D USE_OLD_PETSC:BOOL=OFF \ + prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ - -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc.gitlab" \ - -D PETSC_ARCH:STRING="ubuntu-complex-shared-3.16.6" \ + -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ + -D PETSC_ARCH:STRING="ubuntu-real-shared" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ From 4efdd6bdb16f9713647bdb637f8ebe7e9e40a42f Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 13:12:58 -0700 Subject: [PATCH 015/125] Add smoke test for 240-bus dynamic simulation --- .../dynamic_simulation_full_y/CMakeLists.txt | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index e308b5318..051a749fc 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2017-12-08 09:39:48 d3g096 +# Last Change: 2023-08-28 13:11:06 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -114,6 +114,16 @@ add_custom_command( DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" ) +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" + COMMAND ${CMAKE_COMMAND} + -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" + -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" + -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" + DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + ) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -148,6 +158,14 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -161,6 +179,9 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml ${GRIDPACK_DATA_DIR}/raw/bus3000_gen_no0imp_v23_pslf.raw ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr + ${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml + ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr + ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw ) add_dependencies(dsf.x dsf.x.input) @@ -197,5 +218,6 @@ install(TARGETS dsf2.x DESTINATION bin) # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- -gridpack_add_run_test("dynamic_simulation_full_y" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) From a0a0869faba7875c8f7f61031706d71fcf44ace9 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 13:39:31 -0700 Subject: [PATCH 016/125] Add a CMake macro to do the LU solver check and use it --- .../dynamic_simulation_full_y/CMakeLists.txt | 75 +++++++------------ src/cmake-modules/GridPACK.cmake | 23 +++++- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 051a749fc..c94db802a 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 13:11:06 d3g096 +# Last Change: 2023-08-28 13:21:26 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -74,55 +74,30 @@ add_executable(dsf2.x target_link_libraries(dsf.x ${target_libraries}) target_link_libraries(dsf2.x ${target_libraries}) -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" - ) - -add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" - COMMAND ${CMAKE_COMMAND} - -D INPUT:PATH="${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" - -D OUTPUT:PATH="${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" - -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" - -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" - DEPENDS "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" - ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_145.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_145.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_9b3g.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_9b3g.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_300_cmpld.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_300_cmpld.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_3000.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_3000.xml" +) + +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_240bus.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" +) add_custom_target(dsf.x.input diff --git a/src/cmake-modules/GridPACK.cmake b/src/cmake-modules/GridPACK.cmake index bd07c7ebe..8c0dc6c80 100644 --- a/src/cmake-modules/GridPACK.cmake +++ b/src/cmake-modules/GridPACK.cmake @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created June 10, 2013 by William A. Perkins -# Last Change: 2019-08-16 13:41:26 d3g096 +# Last Change: 2023-08-28 13:18:39 d3g096 # ------------------------------------------------------------- @@ -163,3 +163,24 @@ function(gridpack_add_run_test test_name test_target test_input) gridpack_add_parallel_run_test("${test_name}" ${test_target} "${test_input}") endif () endfunction(gridpack_add_run_test) + + +# ------------------------------------------------------------- +# gridpack_set_lu_solver +# +# This takes a GridPACK input XML file, looks for any occurance of +# "superlu_dist" and changes it to "mumps" if SuperLU_DIST is not +# available in the build. The result is copied to the specified +# output. +# ------------------------------------------------------------- +macro(gridpack_set_lu_solver inputxml outputxml) +add_custom_command( + OUTPUT "${outputxml}" + COMMAND ${CMAKE_COMMAND} + -D INPUT:PATH="${inputxml}" + -D OUTPUT:PATH="${outputxml}" + -D PKG:STRING="${GRIDPACK_MATSOLVER_PKG}" + -P "${PROJECT_SOURCE_DIR}/cmake-modules/set_lu_solver_pkg.cmake" + DEPENDS "${inputxml}" +) +endmacro() From b7135058c25abab8ed5b48abbbe9d98c36facc3a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Mon, 28 Aug 2023 14:30:18 -0700 Subject: [PATCH 017/125] Add smoke test for 2-area dynamic simulation --- src/CMakeLists.txt | 65 +++++++++---------- .../dynamic_simulation_full_y/CMakeLists.txt | 19 +++++- 2 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 729327468..17c20e67d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 3, 2013 by William A. Perkins -# Last Change: 2022-10-05 09:26:46 d3g096 +# Last Change: 2023-08-28 13:46:15 d3g096 # ------------------------------------------------------------- # @@ -313,44 +313,43 @@ endif() message(STATUS "Using PETSc version ${PETSC_VERSION}") if (NOT USE_OLD_PETSC) -if(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") - set(petscconf "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") -elseif(EXISTS "${PETSC_DIR}/include/petscconf.h") - set(petscconf "${PETSC_DIR}/include/petscconf.h") - else() - message(FATAL_ERROR "Could not find PETSc configuration header file + if(EXISTS "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") + set(petscconf "${PETSC_DIR}/${PETSC_ARCH}/include/petscconf.h") + elseif(EXISTS "${PETSC_DIR}/include/petscconf.h") + set(petscconf "${PETSC_DIR}/include/petscconf.h") + else() + message(FATAL_ERROR "Could not find PETSc configuration header file petscconf.h") -endif() + endif() -# checks -# define PETSc variables -include(CheckSymbolExists) -check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) -check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) -check_symbol_exists(PETSC_USE_COMPLEX ${petscconf} PETSC_USE_COMPLEX) -check_symbol_exists(PETSC_LANGUAGE_CXX ${petscconf} PETSC_CLANGUAGE_Cxx) -check_symbol_exists(PETSC_HAVE_SUPERLU_DIST ${petscconf} -PETSC_HAVE_SUPERLU_DIST) -check_symbol_exists(PETSC_HAVE_MUMPS ${petscconf} PETSC_HAVE_MUMPS) + # checks + # define PETSc variables + include(CheckSymbolExists) + check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) + check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) + check_symbol_exists(PETSC_USE_COMPLEX ${petscconf} PETSC_USE_COMPLEX) + check_symbol_exists(PETSC_LANGUAGE_CXX ${petscconf} PETSC_CLANGUAGE_Cxx) + check_symbol_exists(PETSC_HAVE_SUPERLU_DIST ${petscconf} + PETSC_HAVE_SUPERLU_DIST) + check_symbol_exists(PETSC_HAVE_MUMPS ${petscconf} PETSC_HAVE_MUMPS) else() -if (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") -elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") -elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") - include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") -else() - message(FATAL_ERROR "PETSc found, but CMake configuration for PETSc installation not found?") -endif() - -message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") + if (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/conf/PETScConfig.cmake") + elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScConfig.cmake") + elseif (EXISTS "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") + include("${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/PETScBuildInternal.cmake") + else() + message(FATAL_ERROR "PETSc found, but CMake configuration for PETSc installation not found?") + endif() + message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") -# checks + # checks -if (NOT PETSC_HAVE_MPI) - message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") -endif() + if (NOT PETSC_HAVE_MPI) + message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") + endif() endif() # PETSc can be compiled for double precsion (--with-precision=double) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index c94db802a..83a362b89 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 13:21:26 d3g096 +# Last Change: 2023-08-28 14:29:22 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -99,6 +99,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml" ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_twoarea.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml" +) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -141,6 +146,14 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -157,6 +170,9 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_240bus.xml ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw + ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr + ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw ) add_dependencies(dsf.x dsf.x.input) @@ -195,4 +211,5 @@ install(TARGETS dsf2.x DESTINATION bin) # ------------------------------------------------------------- gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) +gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) From f99aeaca54c26fe4257171681c1e4c9d957cd36e Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 06:01:46 -0700 Subject: [PATCH 018/125] Add smoke test for two area case w/ renewables --- .../input/ds/input_twoarea_renewable_mech.xml | 4 ++-- .../dynamic_simulation_full_y/CMakeLists.txt | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml b/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml index f35697b5c..52b1ab687 100644 --- a/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml +++ b/src/applications/data_sets/input/ds/input_twoarea_renewable_mech.xml @@ -1,7 +1,7 @@ - Benchmark_twoarea_v33.raw + kundur-twoarea_v33.raw 50 1.0e-6 @@ -53,7 +53,7 @@ - Benchmark_twoarea_4renewable_mech.dyr + kundur-twoarea_4renewable_mech.dyr 20 0.005 diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 83a362b89..7e8b86a1d 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-28 14:29:22 d3g096 +# Last Change: 2023-08-30 06:00:46 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -104,6 +104,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml" ) +gridpack_set_lu_solver( + "${GRIDPACK_DATA_DIR}/input/ds/input_twoarea_renewable_mech.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_twoarea_renewable_mech.xml" +) + add_custom_target(dsf.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -154,6 +159,10 @@ add_custom_target(dsf.x.input ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -173,6 +182,8 @@ add_custom_target(dsf.x.input ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea.xml ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw + ${CMAKE_CURRENT_BINARY_DIR}/input_twoarea_renewable_mech.xml + ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr ) add_dependencies(dsf.x dsf.x.input) @@ -212,4 +223,5 @@ install(TARGETS dsf2.x DESTINATION bin) gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) +gridpack_add_run_test("dynamic_simulation_two_area_renewable" dsf.x input_twoarea_renewable_mech.xml) From e4c21716ea4afbdf3f72dea2bdb493abba000370 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 09:59:25 -0700 Subject: [PATCH 019/125] Remove math::transpose(Matrix<>, Matrix<>) because of PETSc transpose semantics --- src/math/matrix.hpp | 22 +++++++++++----------- src/math/test/matrix_test.cpp | 24 ++++++++++++------------ src/math/test/matrix_transpose_test.cpp | 5 ++--- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/math/matrix.hpp b/src/math/matrix.hpp index 9ea04d009..3b4c58be3 100644 --- a/src/math/matrix.hpp +++ b/src/math/matrix.hpp @@ -9,7 +9,7 @@ /** * @file matrix.hpp * @author William A. Perkins - * @date 2015-08-18 14:09:32 d3g096 + * @date 2023-08-30 07:58:18 d3g096 * * @brief Declaration of the Matrix class. */ @@ -482,16 +482,16 @@ void add(const MatrixT& A, const MatrixT& B, MatrixT& result) result.add(B); } -/// Make the transpose of a Matrix and put it in another -/** - * - * - * @param A - * @param result - */ -template -void -transpose(const MatrixT& A, MatrixT& result); +// /// Make the transpose of a Matrix and put it in another +// /** +// * +// * +// * @param A +// * @param result +// */ +// template +// void +// transpose(const MatrixT& A, MatrixT& result); /// Get a column from the Matrix and put in specified Vector /** diff --git a/src/math/test/matrix_test.cpp b/src/math/test/matrix_test.cpp index 94f6d20d1..d398f35dd 100644 --- a/src/math/test/matrix_test.cpp +++ b/src/math/test/matrix_test.cpp @@ -8,7 +8,7 @@ /** * @file matrix_test.cpp * @author William A. Perkins - * @date 2016-12-16 09:35:46 d3g096 + * @date 2023-08-30 07:59:49 d3g096 * * @brief Unit tests for Matrix * @@ -990,15 +990,15 @@ BOOST_AUTO_TEST_CASE( NonSquareTranspose ) // FIXME: check B contents - boost::scoped_ptr - C(new TestMatrixType(world, 3, 2, the_storage_type)); - transpose(*A, *C); - C->print(); + // boost::scoped_ptr + // C(new TestMatrixType(world, 3, 2, the_storage_type)); + // transpose(*A, *C); + // C->print(); // FIXME: check C contents - C.reset(A->clone()); - BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); + // C.reset(A->clone()); + // BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); } BOOST_AUTO_TEST_CASE( AnotherNonSquareTranspose ) @@ -1098,13 +1098,13 @@ BOOST_AUTO_TEST_CASE( AnotherNonSquareTranspose ) throw gridpack::Exception("Unknown Matrix storage type"); } - transpose(*A, *C); - C->print(); + // transpose(*A, *C); + // C->print(); - // FIXME: check C contents + // // FIXME: check C contents - C.reset(A->clone()); - BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); + // C.reset(A->clone()); + // BOOST_CHECK_THROW(transpose(*A, *C), gridpack::Exception); } diff --git a/src/math/test/matrix_transpose_test.cpp b/src/math/test/matrix_transpose_test.cpp index 4b8800b46..ffba0d49c 100644 --- a/src/math/test/matrix_transpose_test.cpp +++ b/src/math/test/matrix_transpose_test.cpp @@ -8,7 +8,7 @@ /** * @file matrix_transpose_test.cpp * @author William A. Perkins - * @date 2018-12-18 09:31:40 d3g096 + * @date 2023-08-30 08:02:21 d3g096 * * @brief Unit tests for Matrix * @@ -173,8 +173,7 @@ BOOST_AUTO_TEST_CASE( TransposeRandom ) // see if the transpose is reversible (transpose of transpose needs // to be distributed the same as A - boost::scoped_ptr C(A->clone()); - gridpack::math::transpose(*B, *C); + boost::scoped_ptr C(gridpack::math::transpose(*B)); C->scale(-1.0); boost::scoped_ptr E(gridpack::math::add(*A, *C)); From 6580fda51baae6983ddb412c83c6e03d2c281531 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 30 Aug 2023 13:06:02 -0700 Subject: [PATCH 020/125] Add smoke tests for dsf2.x (same as for dsf.x). --- src/applications/dynamic_simulation_full_y/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index 7e8b86a1d..b21738753 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-30 06:00:46 d3g096 +# Last Change: 2023-08-30 10:04:14 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -221,7 +221,11 @@ install(TARGETS dsf2.x DESTINATION bin) # run application as test # ------------------------------------------------------------- gridpack_add_run_test("dynamic_simulation_full_y_145_bus" dsf.x input_145.xml) +gridpack_add_run_test("dynamic_simulation_full_y_2_145_bus" dsf2.x input_145.xml) gridpack_add_run_test("dynamic_simulation_full_y_240_bus" dsf.x input_240bus.xml) +gridpack_add_run_test("dynamic_simulation_full_y2_240_bus" dsf2.x input_240bus.xml) gridpack_add_run_test("dynamic_simulation_two_area" dsf.x input_twoarea.xml) +gridpack_add_run_test("dynamic_simulation_2_two_area" dsf2.x input_twoarea.xml) gridpack_add_run_test("dynamic_simulation_two_area_renewable" dsf.x input_twoarea_renewable_mech.xml) +gridpack_add_run_test("dynamic_simulation_2_two_area_renewable" dsf2.x input_twoarea_renewable_mech.xml) From e7ab92bebf5b94854509b51a84aefce028310ed1 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 08:46:38 -0700 Subject: [PATCH 021/125] Add smoke test for Wind DSA 9-bus case --- .../development/wind_dsa/CMakeLists.txt | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index 253770f61..c9483529f 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2019-08-16 13:51:41 d3g096 +# Last Change: 2023-08-31 08:45:11 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -81,16 +81,13 @@ target_link_libraries(wind.x ${target_libraries}) target_link_libraries(wind2.x ${target_libraries}) -# Put files necessary to run powerflow2 in binary directory. -# gridpack.petscrc is temporary -- it will be incorporated into -# input.xml +gridpack_set_lu_solver( + "${CMAKE_CURRENT_SOURCE_DIR}/input.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input.xml" +) add_custom_target(wind.x.input - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_SOURCE_DIR}/input.xml - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw ${CMAKE_CURRENT_BINARY_DIR} @@ -112,13 +109,12 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_BINARY_DIR} DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/input.xml + ${CMAKE_CURRENT_BINARY_DIR}/input.xml ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_faults.xml ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_classical.dyr ${CMAKE_CURRENT_SOURCE_DIR}/load.txt ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt - ) add_dependencies(wind.x wind.x.input) add_dependencies(wind2.x wind.x.input) @@ -126,8 +122,10 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- # run applications as tests # ------------------------------------------------------------- -# FIXME: temporarily disabled -#gridpack_add_run_test("wind" wind.x input.xml) +gridpack_add_run_test("wind_dsa" wind.x input.xml) +gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) +gridpack_add_run_test("wind_dsa" wind.x input.xml) +gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) # ------------------------------------------------------------- # install as a sample application From 5b38344f80535a35c4d5761d923d677405c473d9 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 08:57:09 -0700 Subject: [PATCH 022/125] Fix typos in Wind DSA TAMU 2000 case --- src/applications/development/wind_dsa/input_tamu2000_dsf.xml | 2 +- src/applications/development/wind_dsa/tamu_2000_faults.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/input_tamu2000_dsf.xml b/src/applications/development/wind_dsa/input_tamu2000_dsf.xml index afa923b46..3246eba51 100644 --- a/src/applications/development/wind_dsa/input_tamu2000_dsf.xml +++ b/src/applications/development/wind_dsa/input_tamu2000_dsf.xml @@ -51,7 +51,7 @@ 0.005 texas_2k_scenfile_5_full.txt - tamu_2000_faults.xml + tamu_2000_faults.xml 1048 diff --git a/src/applications/development/wind_dsa/tamu_2000_faults.xml b/src/applications/development/wind_dsa/tamu_2000_faults.xml index 55df4d1a8..5366ba2c0 100644 --- a/src/applications/development/wind_dsa/tamu_2000_faults.xml +++ b/src/applications/development/wind_dsa/tamu_2000_faults.xml @@ -1,5 +1,5 @@ - + @@ -16,4 +16,4 @@ - + From 7514dccbc6dcdfb03cde96129d1a1cf38323608a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Thu, 31 Aug 2023 11:00:53 -0700 Subject: [PATCH 023/125] Add a smoke test for Wind DSA TAMU 2000 case --- .../development/wind_dsa/CMakeLists.txt | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index c9483529f..058257bd8 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-31 08:45:11 d3g096 +# Last Change: 2023-08-31 09:05:17 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -86,6 +86,11 @@ gridpack_set_lu_solver( "${CMAKE_CURRENT_BINARY_DIR}/input.xml" ) +gridpack_set_lu_solver( + "${CMAKE_CURRENT_SOURCE_DIR}/input_tamu2000_dsf.xml" + "${CMAKE_CURRENT_BINARY_DIR}/input_tamu2000_dsf.xml" +) + add_custom_target(wind.x.input COMMAND ${CMAKE_COMMAND} -E copy @@ -108,6 +113,22 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_hr_4882_mod_33.raw + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_v23_fullsys_noac_wind_mod.dyr + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/tamu_2000_faults.xml + ${CMAKE_CURRENT_BINARY_DIR} + + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/texas_2k_scenfile_5_full.txt + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input.xml ${CMAKE_CURRENT_SOURCE_DIR}/IEEE3G9B_V23.raw @@ -115,6 +136,11 @@ add_custom_target(wind.x.input ${CMAKE_CURRENT_SOURCE_DIR}/3g9b_classical.dyr ${CMAKE_CURRENT_SOURCE_DIR}/load.txt ${CMAKE_CURRENT_SOURCE_DIR}/wind.txt + ${CMAKE_CURRENT_BINARY_DIR}/input_tamu2000_dsf.xml + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_hr_4882_mod_33.raw + ${CMAKE_CURRENT_SOURCE_DIR}/texas2000_v23_fullsys_noac_wind_mod.dyr + ${CMAKE_CURRENT_SOURCE_DIR}/tamu_2000_faults.xml + ${CMAKE_CURRENT_SOURCE_DIR}/texas_2k_scenfile_5_full.txt ) add_dependencies(wind.x wind.x.input) add_dependencies(wind2.x wind.x.input) @@ -124,8 +150,8 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- gridpack_add_run_test("wind_dsa" wind.x input.xml) gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) -gridpack_add_run_test("wind_dsa" wind.x input.xml) -gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) +gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) +gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) # ------------------------------------------------------------- # install as a sample application From f84ae13fe42e8fb30a3eaa313dc0708c81877280 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 6 Sep 2023 06:57:30 -0700 Subject: [PATCH 024/125] Remove ancient test from `applications/development/dynamic_simulation_reduced_y` --- .../development/dynamic_simulation_reduced_y/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt b/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt index fd0b11f4c..ddd838b4a 100644 --- a/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt +++ b/src/applications/development/dynamic_simulation_reduced_y/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2019-01-04 08:39:22 d3g096 +# Last Change: 2023-09-06 06:50:50 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -100,5 +100,5 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.install.in # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- -gridpack_add_run_test("dynamic_simulation" ds.x input.xml) +# gridpack_add_run_test("dynamic_simulation" ds.x input.xml) From 7ed20ef898370766d5c139de03b8a06a26243863 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:15:20 -0500 Subject: [PATCH 025/125] Fix env var --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index fdadabf24..38b96e05f 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -106,7 +106,7 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GRIDPACK_EXT_DEPS:?}" "$build_dir" "$install_dir" +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" install_gridpack_python "$build_dir" "$install_dir" echo "Completed GridPACK installation" From d3aecd995bf6fbf78492df9d9455ec6d1d88558c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:15:24 -0500 Subject: [PATCH 026/125] Set number of make threads dynamically --- install_gridpack.sh | 2 +- install_gridpack_deps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 38b96e05f..3d0f53d27 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -50,7 +50,7 @@ function install_gridpack { # install echo "Installing GridPACK" - make -j 10 install + make -j "$(nproc)" install popd || exit diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 74a5dd15a..98a6bd29e 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -90,7 +90,7 @@ function install_ga { # install echo "Installing Global Arrays" - make -j 10 install >../log/ga_install.log 2>&1 + make -j "$(nproc)" install >../log/ga_install.log 2>&1 popd || exit From 795505d80f4993ee6fbb1004d36611ff8ec72df1 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 6 Sep 2023 10:16:43 -0500 Subject: [PATCH 027/125] Add ci script and container image definition --- .gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ dockerfile | 17 +++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..1b87d0b95 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,44 @@ +default: + + tags: [ basic, gridpack, ikp, k8s ] + + +variables: + + GP_EXT_DEPS: /gridpack-dependencies + LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + + +# https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy +build-container: + + image: + + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + + script: + + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --build-arg http_proxy=$http_proxy + --build-arg https_proxy=$https_proxy + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" + + rules: + + - if: $CI_PIPELINE_SOURCE == "branch" + changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "tag" + + +test: + + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} + + needs: [ build-container ] + + script: + + - ./install_gridpack.sh diff --git a/dockerfile b/dockerfile new file mode 100644 index 000000000..1a0ee60f3 --- /dev/null +++ b/dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:22.04 + +RUN apt update \ + && apt upgrade -y \ + && apt install -y wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + +ENV GP_EXT_DEPS=/gridpack-dependencies +ENV BOOST_VERSION "1.78.0" +ENV GA_VERSION "5.8" +ENV PETSC_VERSION "3.16.4" +ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + +WORKDIR ${GP_EXT_DEPS} + +COPY *.sh . + +RUN ./install_gridpack_deps.sh && rm *.sh From 66fbc23a13bfd5edd55e4cb3809e0339dcedf02a Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 13 Sep 2023 08:53:25 -0700 Subject: [PATCH 028/125] Fix incorrect Matrix type Jacobian supplied to DAESolver constructor --- src/math/dae_solver.hpp | 4 ++-- src/math/dae_solver_functions.hpp | 4 ++-- src/math/dae_solver_implementation.hpp | 13 +++++++++---- src/math/petsc/petsc_dae_solver.cpp | 8 ++++---- src/math/petsc/petsc_dae_solver_implementation.hpp | 4 ++-- src/math/test/dae_solver_test.cpp | 5 ++++- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/math/dae_solver.hpp b/src/math/dae_solver.hpp index 8a7f05067..d13c6c896 100644 --- a/src/math/dae_solver.hpp +++ b/src/math/dae_solver.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver.hpp * @author William A. Perkins - * @date 2019-12-05 07:52:33 d3g096 + * @date 2023-09-13 07:37:27 d3g096 * * @brief * @@ -72,7 +72,7 @@ class DAESolverT DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix* J, + MatrixType* J, JacobianBuilder& jbuilder, FunctionBuilder& fbuilder, EventManagerPtr eman); diff --git a/src/math/dae_solver_functions.hpp b/src/math/dae_solver_functions.hpp index 70cdf64cd..553da0bb2 100644 --- a/src/math/dae_solver_functions.hpp +++ b/src/math/dae_solver_functions.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver_functions.hpp * @author William A. Perkins - * @date 2015-05-07 13:15:18 d3g096 + * @date 2023-09-13 07:56:50 d3g096 * * @brief * @@ -31,7 +31,7 @@ template struct DAEBuilder { typedef VectorT VectorType; - typedef Matrix MatrixType; + typedef MatrixT MatrixType; /// Functions that compute a Jacobian typedef diff --git a/src/math/dae_solver_implementation.hpp b/src/math/dae_solver_implementation.hpp index 33d6373cc..b1484e742 100644 --- a/src/math/dae_solver_implementation.hpp +++ b/src/math/dae_solver_implementation.hpp @@ -9,7 +9,7 @@ /** * @file dae_solver_implementation.hpp * @author William A. Perkins - * @date 2019-12-05 09:15:09 d3g096 + * @date 2023-09-13 08:34:39 d3g096 * * @brief * @@ -57,12 +57,12 @@ class DAESolverImplementation : parallel::Distributed(comm), utility::Configurable("DAESolver"), utility::Uncopyable(), + p_J(new MatrixType(comm,local_size,local_size)), + p_J_allocated(true), p_Fbuilder(fbuilder), p_Jbuilder(jbuilder), p_eventManager(eman), p_doAdaptive(true) - { - p_J = new gridpack::math::Matrix(comm,local_size,local_size); - } + { } DAESolverImplementation(const parallel::Communicator& comm, const int local_size, @@ -74,6 +74,7 @@ class DAESolverImplementation utility::Configurable("DAESolver"), utility::Uncopyable(), p_J(J), + p_J_allocated(false), p_Fbuilder(fbuilder), p_Jbuilder(jbuilder), p_eventManager(eman), p_doAdaptive(true) @@ -84,6 +85,7 @@ class DAESolverImplementation /// Destructor ~DAESolverImplementation(void) { + if (p_J_allocated) delete p_J; } protected: @@ -91,6 +93,9 @@ class DAESolverImplementation /// A matrix to hold the Jacobian MatrixType *p_J; + /// Is p_J locally allocated? + bool p_J_allocated; + /// A function to build the RHS vector FunctionBuilder p_Fbuilder; diff --git a/src/math/petsc/petsc_dae_solver.cpp b/src/math/petsc/petsc_dae_solver.cpp index 51b8982b9..47f2720e7 100644 --- a/src/math/petsc/petsc_dae_solver.cpp +++ b/src/math/petsc/petsc_dae_solver.cpp @@ -9,7 +9,7 @@ /** * @file dae_solver.cpp * @author William A. Perkins - * @date 2019-11-21 07:41:15 d3g096 + * @date 2023-09-13 07:48:08 d3g096 * * @brief * @@ -47,7 +47,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix* J, + DAESolverT::MatrixType* J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman) @@ -71,7 +71,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - gridpack::math::Matrix* J, + DAESolverT::MatrixType* J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman); @@ -86,7 +86,7 @@ DAESolverT::DAESolverT(const parallel::Communicator& comm, template DAESolverT::DAESolverT(const parallel::Communicator& comm, const int local_size, - Matrix *J, + DAESolverT::MatrixType *J, DAESolverT::JacobianBuilder& jbuilder, DAESolverT::FunctionBuilder& fbuilder, DAESolverT::EventManagerPtr eman); diff --git a/src/math/petsc/petsc_dae_solver_implementation.hpp b/src/math/petsc/petsc_dae_solver_implementation.hpp index f10886fda..aa1334ac7 100644 --- a/src/math/petsc/petsc_dae_solver_implementation.hpp +++ b/src/math/petsc/petsc_dae_solver_implementation.hpp @@ -10,7 +10,7 @@ /** * @file petsc_dae_solver_implementation.hpp * @author William A. Perkins - * @date 2023-08-24 09:42:12 d3g096 + * @date 2023-09-13 07:43:15 d3g096 * * @brief * @@ -70,7 +70,7 @@ class PETScDAESolverImplementation PETScDAESolverImplementation(const parallel::Communicator& comm, const int local_size, - Matrix* J, + MatrixType* J, JacobianBuilder& jbuilder, FunctionBuilder& fbuilder, EventManagerPtr eman) diff --git a/src/math/test/dae_solver_test.cpp b/src/math/test/dae_solver_test.cpp index 27a8e075e..7a8208729 100644 --- a/src/math/test/dae_solver_test.cpp +++ b/src/math/test/dae_solver_test.cpp @@ -9,7 +9,7 @@ /** * @file dae_solver_test.cpp * @author William A. Perkins - * @date 2019-12-04 12:40:57 d3g096 + * @date 2023-09-13 07:44:15 d3g096 * * @brief * @@ -169,8 +169,11 @@ class RoberProblem const double& shift, MatrixType& J) { int lo, hi; + int rlo, rhi; X.localIndexRange(lo, hi); + J.localRowRange(rlo, rhi); BOOST_ASSERT((hi-lo) == this->size()); + BOOST_ASSERT((rhi-rlo) == this->size()); std::vector x(this->size()); X.getElementRange(lo, hi, &x[0]); J.setElement(lo+0, lo+0, shift + 0.04); From 2c0744eb8f1a08969d5a76a1036885d99c0b2483 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 13 Sep 2023 09:16:50 -0700 Subject: [PATCH 029/125] Update my build script --- src/example_configuration.sh | 53 ++---------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index afe9e6ffb..f3508ec33 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -145,55 +145,6 @@ elif [ $host == "we32673" ]; then -D CMAKE_INSTALL_PREFIX:PATH="$prefix/gridpack-install" \ $common_flags .. -elif [ $host == "WE30729" ]; then - - # Macbook using CLang 6.0 compilers and MPICH via MacPorts - # Pretty much the same as WE32673 - - CC=/opt/local/bin/clang - export CC - CXX=/opt/local/bin/clang++ - export CXX - - prefix="$HOME/Projects/GridPACK" - - if [ "$shared"x = "ON"x ]; then - pdir="$prefix/petsc-3.10.5" - parch="macosx-complex-c-shared" - else - pdir="$prefix/petsc-3.8.4" - parch="arch-macosx-clang-real-opt" - fi - cmake $options \ - -D BOOST_ROOT:STRING="/opt/local" \ - -D PETSC_DIR:PATH="$pdir" \ - -D PETSC_ARCH:STRING="$parch" \ - -D MPI_CXX_COMPILER:STRING='/opt/local/bin/mpicxx' \ - -D MPI_C_COMPILER:STRING='/opt/local/bin/mpicc' \ - -D MPIEXEC:STRING='/opt/local/bin/mpiexec' \ - -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ - -D GRIDPACK_TEST_TIMEOUT:STRING=60 \ - -D USE_CPLEX:BOOL=OFF \ - -D USE_GLPK:BOOL=ON \ - -D GLPK_ROOT_DIR:PATH="/opt/local" \ - -D CMAKE_INSTALL_PREFIX:PATH="$prefix/gridpack-hadrec" \ - $common_flags .. - -elif [ $host == "olympus.local" ]; then - - prefix="/pic/projects/gridpack/software" - cmake $options \ - -D GA_DIR:STRING="/pic/projects/gridpack/ga-5-2" \ - -D GA_EXTRA_LIBS:STRING="-libverbs" \ - -D BOOST_ROOT:STRING="$prefix" \ - -D PETSC_DIR:STRING="$prefix/petsc-3.4.0" \ - -D PETSC_ARCH:STRING='olympus-openmpi-gnu-cxx-complex-opt' \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - $common_flags .. - - elif [ $host == "constance" ]; then CC=`which gcc` @@ -281,8 +232,8 @@ elif [ $host == "tlaloc" ]; then prefix="$HOME/Projects/GridPakLDRD/gridpack-install" cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ - -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ - -D PETSC_ARCH:STRING="ubuntu-real-shared" \ + -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.16.6" \ + -D PETSC_ARCH:STRING="ubuntu-real-shared-debug" \ -D USE_OLD_PETSC:BOOL=OFF \ -D BOOST_ROOT:PATH="/usr" \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ From 7ac8094e06f4d8ed2d850dc1773c0e0882ee790c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:26:58 -0500 Subject: [PATCH 030/125] Test pipeline trigger --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b87d0b95..c5207349c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,4 +41,5 @@ test: script: - - ./install_gridpack.sh + - echo "hello world" + # - ./install_gridpack.sh From 7744e719120f3f5a65d53dafeafb9cdcbcdc3cea Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:32:51 -0500 Subject: [PATCH 031/125] Build container, make dep from test optional --- .gitlab-ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5207349c..e9093731c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,18 +26,20 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/Dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" - rules: + # rules: - - if: $CI_PIPELINE_SOURCE == "branch" - changes: [ "dockerfile", "install/*" ] - - if: $CI_PIPELINE_SOURCE == "tag" + # - if: $CI_PIPELINE_SOURCE == "branch" + # changes: [ "dockerfile", "install/*" ] + # - if: $CI_PIPELINE_SOURCE == "tag" test: image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} - needs: [ build-container ] + needs: + - job: build-container + optional: true script: From 9cc9d04b11c62f0761fc5a4f2668fd5afa7a4a24 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 14 Sep 2023 10:46:52 -0500 Subject: [PATCH 032/125] Update dockerfile path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e9093731c..55c2e2bf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ build-container: --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy - --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" # rules: From 09dd060ec902048380231ab5e1f7feb2cc81f2a7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 11:26:41 -0500 Subject: [PATCH 033/125] Increase k8s memory limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 55c2e2bf2..43d09ce3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - + KUBERNETES_MEMORY_LIMIT: 4Gi # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: From 1a5b94a892b2676a985f9d5918e62bd2ecd2e748 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 12:39:43 -0500 Subject: [PATCH 034/125] Increase mem request/limit and job timeout --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43d09ce3e..fdac37d45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,11 +7,14 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_MEMORY_LIMIT: 4Gi + KUBERNETES_MEMORY_REQUEST: 4Gi + KUBERNETES_MEMORY_LIMIT: 6Gi # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: + timeout: 3 hours + image: name: gcr.io/kaniko-project/executor:v1.9.0-debug From 60572f0f775fce7e5fd04b5944ead667284ae84a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 27 Sep 2023 15:51:45 -0500 Subject: [PATCH 035/125] Add cpu and increase timeout further --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdac37d45..19308aff4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,13 +7,17 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_MEMORY_REQUEST: 4Gi - KUBERNETES_MEMORY_LIMIT: 6Gi + + KUBERNETES_CPU_REQUEST: "500m" + KUBERNETES_CPU_LIMIT: "1" + KUBERNETES_MEMORY_REQUEST: "4Gi" + KUBERNETES_MEMORY_LIMIT: "6Gi" + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: - timeout: 3 hours + timeout: 5 hours image: From 50adb0d824992b147fe79842a281a4fdc82cce82 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 11:49:47 -0500 Subject: [PATCH 036/125] Troubleshooting env --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19308aff4..ae5e17b0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ build-container: script: + - env - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy From 8d052b2064bfe643e6c8342d4e3d71d0ee1cd91d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 11:59:13 -0500 Subject: [PATCH 037/125] Use correct predefined CI variable --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae5e17b0b..0f36dde68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,13 +26,12 @@ build-container: script: - - env - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF}" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" # rules: @@ -43,7 +42,7 @@ build-container: test: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF} + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} needs: - job: build-container From e0cecaafb966c3c00ca370badccbcada6c3aef25 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 2 Oct 2023 15:06:36 -0500 Subject: [PATCH 038/125] Add conditions for building container And enable real test job --- .gitlab-ci.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f36dde68..da56af30d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,11 +33,11 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" - # rules: + rules: - # - if: $CI_PIPELINE_SOURCE == "branch" - # changes: [ "dockerfile", "install/*" ] - # - if: $CI_PIPELINE_SOURCE == "tag" + - if: $CI_PIPELINE_SOURCE == "branch" + changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "tag" test: @@ -50,5 +50,4 @@ test: script: - - echo "hello world" - # - ./install_gridpack.sh + - ./install_gridpack.sh From 4d60837a5da16bfa6302d6a52096280be9193eed Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 09:02:16 -0500 Subject: [PATCH 039/125] Adjust k8s parameters and gitlab job timeout --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da56af30d..878580f0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,10 +8,10 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "500m" - KUBERNETES_CPU_LIMIT: "1" + KUBERNETES_CPU_REQUEST: "2" + KUBERNETES_CPU_LIMIT: "4" KUBERNETES_MEMORY_REQUEST: "4Gi" - KUBERNETES_MEMORY_LIMIT: "6Gi" + KUBERNETES_MEMORY_LIMIT: "8Gi" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -40,11 +40,14 @@ build-container: - if: $CI_PIPELINE_SOURCE == "tag" -test: +build-gridpack: + + timeout: 5 hours image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} needs: + - job: build-container optional: true From 503dabdc508bc26d8bde2009c7a7265bf58d6ef2 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 09:09:48 -0500 Subject: [PATCH 040/125] Update CPU limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 878580f0e..c7954635a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} KUBERNETES_CPU_REQUEST: "2" - KUBERNETES_CPU_LIMIT: "4" + KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From a07da9a0fc3ca5b2c5902ed44bf534406015c8b8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 10:08:16 -0500 Subject: [PATCH 041/125] Adjust CPU req/limits --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7954635a..754b59834 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,8 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "2" - KUBERNETES_CPU_LIMIT: "2" + KUBERNETES_CPU_REQUEST: "1500m" + KUBERNETES_CPU_LIMIT: "6" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 3f0fca80874bae699ebd53d7d690ca239d4730d6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 10:14:38 -0500 Subject: [PATCH 042/125] Adjust CPU limit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 754b59834..128ebf2f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} KUBERNETES_CPU_REQUEST: "1500m" - KUBERNETES_CPU_LIMIT: "6" + KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 263135fc745c9e3cf12ff3540b61163e5b398b7f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 11:57:28 -0500 Subject: [PATCH 043/125] Adjust cpu request --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 128ebf2f3..3af2de4ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: GP_EXT_DEPS: /gridpack-dependencies LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "1500m" + KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" From 7d97be1814deb2f88978ac75c3462152ac3d5f36 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 15:47:06 -0500 Subject: [PATCH 044/125] Set GRIDPACK_DIR --- install_gridpack.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 3d0f53d27..1cb799cbc 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,6 +1,6 @@ #! /bin/bash -# installs GridPACK and its python wrappter +# installs GridPACK and its python wrapper # gridPACK is built in src/build and installed to src/install # run this from the top level GridPACK directory @@ -71,7 +71,10 @@ function install_gridpack_python { pushd python || exit - export RHEL_OPENMPI_HACK=yes + os_id=$(source /etc/os-release; echo "$ID") + if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then + export RHEL_OPENMPI_HACK=yes + fi # set python executable path python_exe=$(which python || which python3) @@ -79,6 +82,9 @@ function install_gridpack_python { # remove existing build dir rm -rf build + # export GRIDPACK_DIR + export GRIDPACK_DIR="${gridpack_install_dir}" + # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build @@ -93,7 +99,7 @@ function install_gridpack_python { # install echo "Installing GridPACK python wrapper" - ${python_exe} setup.py install --home="$GRIDPACK_DIR" + ${python_exe} setup.py install --home="$gridpack_install_dir" popd || exit From 8ba354292847c80e6cf12984a6668128cf7600ae Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 3 Oct 2023 16:41:20 -0500 Subject: [PATCH 045/125] Uncomment gp_include and gp_libs --- src/lib/GridPACK.cmake.in | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index eaa59dc56..c08a55776 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -233,15 +233,15 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() -# list(APPEND gp_include -# @Boost_INCLUDE_DIR@ -# @MPI_INCLUDE_PATH@ -# ) - -# list(APPEND gp_libs -# @Boost_LIBRARIES@ -# @MPI_CXX_LIBRARIES@ -# ) + list(APPEND gp_include + @Boost_INCLUDE_DIR@ + @MPI_INCLUDE_PATH@ + ) + + list(APPEND gp_libs + @Boost_LIBRARIES@ + @MPI_CXX_LIBRARIES@ + ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 9b533f022e8c56bd9b4aed1348f86374aa1e979b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 12:38:31 -0500 Subject: [PATCH 046/125] Add Boost_NO_BOOST_CMAKE --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 1cb799cbc..56396e7d9 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -46,6 +46,7 @@ function install_gridpack { -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ .. # install From 48d1adc4f48c1ce8edb6f9151a1e86fdb9bcad2e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 13:27:01 -0500 Subject: [PATCH 047/125] Set max number of make jobs --- .gitlab-ci.yml | 4 ++++ install_gridpack.sh | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3af2de4ca..b7082ee9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,6 +46,10 @@ build-gridpack: image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + variables: + + MAKE_JOBS: "2" + needs: - job: build-container diff --git a/install_gridpack.sh b/install_gridpack.sh index 56396e7d9..a0dfa5031 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -51,7 +51,7 @@ function install_gridpack { # install echo "Installing GridPACK" - make -j "$(nproc)" install + make -j "${MAKE_JOBS:-$(nproc)}" install popd || exit @@ -72,7 +72,10 @@ function install_gridpack_python { pushd python || exit - os_id=$(source /etc/os-release; echo "$ID") + os_id=$( + source /etc/os-release + echo "$ID" + ) if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then export RHEL_OPENMPI_HACK=yes fi From e88724d5a942c6fc491f3ecfeb9dfde3be15d524 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 4 Oct 2023 14:07:03 -0500 Subject: [PATCH 048/125] Only append python path if exists --- install_gridpack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index a0dfa5031..072d883c6 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -99,7 +99,7 @@ function install_gridpack_python { mkdir -p "${py_lib}" # add lib to python path - export PYTHONPATH="${py_lib}:${PYTHONPATH}" + export PYTHONPATH="${py_lib}${PYTHONPATH:+:$PYTHONPATH}" # install echo "Installing GridPACK python wrapper" From a442f45ee3d0cadd1f0266bc5d877192aec4c597 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 11:33:24 -0500 Subject: [PATCH 049/125] Add comment --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 072d883c6..a509fdfd9 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -72,6 +72,7 @@ function install_gridpack_python { pushd python || exit + # set an env var if we are running on RHEL os_id=$( source /etc/os-release echo "$ID" From 1107658c42ed387be1ece8d4b28c66b90cda2052 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 11:33:37 -0500 Subject: [PATCH 050/125] Add test job --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7082ee9b..f23bd8591 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,3 +58,19 @@ build-gridpack: script: - ./install_gridpack.sh + + + artifacts: + + untracked: true + + +test-gridpack: + + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + + needs: [ build-gridpack ] + + script: + + - make test From 725cf47ffcac4c8f4b0014d2141f100d6dd03957 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 11 Oct 2023 12:01:06 -0500 Subject: [PATCH 051/125] Use default timeout for build --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f23bd8591..db5bae442 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,8 +42,6 @@ build-container: build-gridpack: - timeout: 5 hours - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} variables: From 920d93e41366ec517f746497fa91aa4d6876bb84 Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 11 Oct 2023 10:40:29 -0700 Subject: [PATCH 052/125] Remove 2000-bus Wind-DSA smoke tests - they take too long --- src/applications/development/wind_dsa/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/applications/development/wind_dsa/CMakeLists.txt b/src/applications/development/wind_dsa/CMakeLists.txt index 058257bd8..315e8c2d9 100644 --- a/src/applications/development/wind_dsa/CMakeLists.txt +++ b/src/applications/development/wind_dsa/CMakeLists.txt @@ -10,7 +10,7 @@ # ------------------------------------------------------------- # ------------------------------------------------------------- # Created May 6, 2013 by William A. Perkins -# Last Change: 2023-08-31 09:05:17 d3g096 +# Last Change: 2023-10-11 10:33:21 d3g096 # ------------------------------------------------------------- set(target_libraries @@ -150,8 +150,8 @@ add_dependencies(wind2.x wind.x.input) # ------------------------------------------------------------- gridpack_add_run_test("wind_dsa" wind.x input.xml) gridpack_add_run_test("wind_dsa_2" wind2.x input.xml) -gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) -gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) +# gridpack_add_run_test("wind_dsa_tamu2000" wind.x input_tamu2000_dsf.xml) +# gridpack_add_run_test("wind_dsa_2_tamu2000" wind2.x input_tamu2000_dsf.xml) # ------------------------------------------------------------- # install as a sample application From f5e91bc8278ca99351408db4424b78fee322ffcf Mon Sep 17 00:00:00 2001 From: William Perkins Date: Wed, 11 Oct 2023 10:41:32 -0700 Subject: [PATCH 053/125] Switch to a direct linear solver because this was diverging --- .../development/wind_dsa/input.xml | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/applications/development/wind_dsa/input.xml b/src/applications/development/wind_dsa/input.xml index 4a6b36bd1..d290f615c 100644 --- a/src/applications/development/wind_dsa/input.xml +++ b/src/applications/development/wind_dsa/input.xml @@ -51,11 +51,10 @@ 1.0E-08 50 - -ksp_type bicg - -pc_type bjacobi - -sub_pc_type ilu -sub_pc_factor_levels 5 -sub_ksp_type preonly - + -ksp_type richardson + -pc_type lu + -pc_factor_mat_solver_type superlu_dist + -ksp_max_it 1 @@ -64,13 +63,10 @@ 1.0E-05 50 - -ksp_type bicg - -pc_type bjacobi - -sub_pc_type ilu -sub_pc_factor_levels 5 -sub_ksp_type preonly - + -ksp_type richardson + -pc_type lu + -pc_factor_mat_solver_type superlu_dist + -ksp_max_it 1 @@ -110,7 +106,7 @@ -ksp_type richardson -pc_type lu - -pc_factor_mat_solver_type petsc + -pc_factor_mat_solver_type superlu_dist -pc_factor_shift_type NONZERO -ksp_max_it 1 From d7d8907fa6ae22275c69b9814cb15484e5b1629a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 16 Oct 2023 13:18:21 -0500 Subject: [PATCH 054/125] Whitespace change --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db5bae442..4099b5be0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,6 @@ build-gridpack: - ./install_gridpack.sh - artifacts: untracked: true From 1231f64ca1c4ee7c68f3d01d6c4f732c03b2f895 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 16 Oct 2023 13:56:36 -0500 Subject: [PATCH 055/125] Troubleshooting --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4099b5be0..9c1663e18 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,4 +70,5 @@ test-gridpack: script: + - find . -type d -maxdepth 4 - make test From d4c0929badb81ebaa0dc85e6e53d1dc4264cabbc Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 01:01:06 -0500 Subject: [PATCH 056/125] Revert block comment --- src/lib/GridPACK.cmake.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 2d47cbe57..e32e09989 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -238,10 +238,10 @@ function(gridpack_setup) @MPI_INCLUDE_PATH@ ) -# list(APPEND gp_libs -# @Boost_LIBRARIES@ -# @MPI_CXX_LIBRARIES@ -# ) + list(APPEND gp_libs + @Boost_LIBRARIES@ + @MPI_CXX_LIBRARIES@ + ) set(GRIDPACK_INCLUDE_DIRS ${gp_include} From 9f7746186c2727045c8eafa28277a3875576af48 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 01:01:22 -0500 Subject: [PATCH 057/125] Fix test command and save results --- .gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c1663e18..c31219db9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,5 +70,10 @@ test-gridpack: script: - - find . -type d -maxdepth 4 - - make test + - make -C src/build test + + artifacts: + + paths: + + - src/build/Testing/Temporary From 1fd0ea7bfa57934e3ec23a07b87971803553dd02 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 02:06:58 -0500 Subject: [PATCH 058/125] Always save artifacts for the test job --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c31219db9..538c9375d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,3 +77,5 @@ test-gridpack: paths: - src/build/Testing/Temporary + + when: always From ea037d83defdb82a85c7c98cef3f9885ceed92d8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 03:02:52 -0500 Subject: [PATCH 059/125] Remove version numbers from LD_LIBRARY_PATH --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 1a0ee60f3..72a728933 100644 --- a/dockerfile +++ b/dockerfile @@ -8,7 +8,7 @@ ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" ENV GA_VERSION "5.8" ENV PETSC_VERSION "3.16.4" -ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} WORKDIR ${GP_EXT_DEPS} From 098a8465f95bfd77386fe4d8ca5c36179258858f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 03:11:01 -0500 Subject: [PATCH 060/125] Run tests as non-root user --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 538c9375d..ebd5bef07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,9 @@ test-gridpack: script: - - make -C src/build test + - useradd gridpack + - chown -R gridpack:gridpack . + - su gridpack -c 'make -C src/build test' artifacts: From 22b88f52e1b47b41ccb219881e30d0c0f0ae1bf5 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 17 Oct 2023 07:58:27 -0500 Subject: [PATCH 061/125] Fix rules for build-container job --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebd5bef07..d1096dbe1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,9 +35,10 @@ build-container: rules: - - if: $CI_PIPELINE_SOURCE == "branch" - changes: [ "dockerfile", "install/*" ] + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile" ] - if: $CI_PIPELINE_SOURCE == "tag" + - when: manual build-gridpack: From ddf9948cfdcd253242309a864618dd4631356bd8 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:49:29 -0500 Subject: [PATCH 062/125] Don't override env vars --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1096dbe1..d482146ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,9 +5,6 @@ default: variables: - GP_EXT_DEPS: /gridpack-dependencies - LD_LIBRARY_PATH: ${GP_EXT_DEPS}/boost_${BOOST_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-${GA_VERSION}/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} - KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" From fc96b3661b6519c522f36d0460a5f5af9000ac13 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:49:58 -0500 Subject: [PATCH 063/125] Remove manual container build rule --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d482146ac..edfa1d193 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,6 @@ build-container: - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile" ] - if: $CI_PIPELINE_SOURCE == "tag" - - when: manual build-gridpack: From 45b04639b9a5f59a4bc0b3a4caaafe46f5508554 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 23 Oct 2023 06:57:59 -0500 Subject: [PATCH 064/125] Add build status badge --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 08d243154..5b7422099 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ # GridPACKTM--> # GridPACK: High-Performance Electric Grid Simulation +![PNNL Build Status](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.github.com%2Frepos%2FGridOPTICS%2FGridPACK%2Fcommits%2Fdevelop%2Fstatus&query=%24.state&logo=gitlab&label=build%20status&color=white +) + GridPACK is an open-source high-performance (HPC) package for simulation of large-scale electrical grids. Powered by distributed (parallel) computing and high-performance numerical solvers, GridPACK offers several applications forfast simulation of electrical transmission systems. GridPACK includes a number of prebuilt applications that can be directly used. The most commonly used and well-developed are: - AC Power Flow - Dynamics Simulation @@ -18,7 +21,7 @@ In addition, GridPACK is also a framework to simplify the development of new app See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms. ## Usage -See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. +See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. - Quick Guide (To do) @@ -36,16 +39,16 @@ The best (and fastest) way to reach us for any technical questions is by posting ## Citing GridPACK ``` -@article{doi:10.1177/1094342015607609, -author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, -title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, -journal = {The International Journal of High Performance Computing Applications}, -volume = {30}, -number = {2}, -pages = {223-240}, -year = {2016}, -doi = {10.1177/1094342015607609}, -URL = {https://doi.org/10.1177/1094342015607609}, +@article{doi:10.1177/1094342015607609, +author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, +title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, +journal = {The International Journal of High Performance Computing Applications}, +volume = {30}, +number = {2}, +pages = {223-240}, +year = {2016}, +doi = {10.1177/1094342015607609}, +URL = {https://doi.org/10.1177/1094342015607609}, eprint = {https://doi.org/10.1177/1094342015607609} ``` @@ -69,7 +72,7 @@ GridPACK has been developed through funding from various sources over the years. ## Copyright Copyright © 2013, Battelle Memorial Institute. -GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. +GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. See the [license](src/LICENSE.md) file for details. From d71c50cf974980e433d3dfcdf92e255908362f93 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 24 Oct 2023 16:11:03 -0500 Subject: [PATCH 065/125] Only use two processors --- install_gridpack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index a509fdfd9..688832d4e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -47,6 +47,7 @@ function install_gridpack { -D BUILD_SHARED_LIBS=YES \ -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ + -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ .. # install From 0fe20f01db64c25f7363c5024d893d7b7cf81e38 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 8 Nov 2023 10:52:48 -0600 Subject: [PATCH 066/125] Show output for failed tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index edfa1d193..bbb6ad26d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,7 +69,7 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - - su gridpack -c 'make -C src/build test' + - su gridpack -c 'ctest --test-dir src/build --output-on-failure' artifacts: From b7003fc5df4cbe7a593e9c88e13a41cf8227f534 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Thu, 28 Dec 2023 15:04:02 -0600 Subject: [PATCH 067/125] Test with multiple distributions --- .gitlab-ci.yml | 26 +++++++++++++++++++++++--- dockerfile | 7 +++---- install_environment_packages.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 install_environment_packages.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bbb6ad26d..b3e38ecb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,9 +11,24 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" +.multi-distro: + + parallel: + + matrix: + - BASE_IMAGE: rockylinux:9 + - BASE_IMAGE: ubuntu:22.04 + + variables: + + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE%%:*}-${CI_COMMIT_REF_NAME}" + + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: + extends: .multi-distro + timeout: 5 hours image: @@ -25,10 +40,11 @@ build-container: - /kaniko/executor --context "${CI_PROJECT_DIR}" + --build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}" + --destination "${IMAGE_PATH}" rules: @@ -39,7 +55,9 @@ build-container: build-gridpack: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + extends: .multi-distro + + image: "${IMAGE_PATH}" variables: @@ -61,7 +79,9 @@ build-gridpack: test-gridpack: - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME} + extends: .multi-distro + + image: ${IMAGE_PATH} needs: [ build-gridpack ] diff --git a/dockerfile b/dockerfile index 72a728933..b40b5f5f6 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,7 @@ -FROM ubuntu:22.04 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} -RUN apt update \ - && apt upgrade -y \ - && apt install -y wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config +RUN install_environment_packages.sh ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" diff --git a/install_environment_packages.sh b/install_environment_packages.sh new file mode 100644 index 000000000..9109cc8d7 --- /dev/null +++ b/install_environment_packages.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +# installs necessary packages for a given distribution + +set -xeuo pipefail + +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +case $distribution in +debian | ubuntu) + apt update && + apt upgrade -y && + apt install -y \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + ;; +fedora | rhel | centos | rocky) + dnf update -y && + dnf install -y \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf + ;; +*) + echo "$distribution not supported" + exit 1 + ;; +esac From 0f5883dcd895629fff6986461397ce1b9bff3c80 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 29 Dec 2023 11:44:37 -0600 Subject: [PATCH 068/125] Back out changes to src --- src/CMakeLists.txt | 50 +++++++++---------- .../dynamic_simulation_full_y/CMakeLists.txt | 40 +++++++-------- src/cmake-modules/GridPACK.cmake | 16 +++--- src/example_configuration.sh | 36 +++++++------ src/lib/GridPACK.cmake.in | 14 +++--- 5 files changed, 81 insertions(+), 75 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 29db21417..9aa459965 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,7 @@ # Created May 3, 2013 by William A. Perkins # ------------------------------------------------------------- -# +# cmake_minimum_required (VERSION 3.5.0) project (GridPACK) @@ -25,7 +25,7 @@ set (GridPACK_VERSION_PATCH 0) enable_language(CXX) # where to look for special .cmake files -list (APPEND CMAKE_MODULE_PATH +list (APPEND CMAKE_MODULE_PATH "${GridPACK_SOURCE_DIR}/cmake-modules" ) include(GridPACK) @@ -39,7 +39,7 @@ endif() # add GOSS directory option (GOSS_DIR "Point to directory with GOSS files" OFF) if (GOSS_DIR) - add_definitions (-DUSE_GOSS=1) + add_definitions (-DUSE_GOSS=1) include_directories(AFTER ${GOSS_DIR}/include/activemq-cpp-3.8.4) set(GOSS_INCLUDE "${GOSS_DIR}/include/activemq-cpp-3.8.4") set (GOSS_LIBRARY "${GOSS_DIR}/lib/libactivemq-cpp.a") @@ -159,14 +159,14 @@ include_directories(AFTER ${MPI_CXX_INCLUDE_PATH}) # set (CMAKE_FIND_LIBRARY_SUFFIXES ".a") #endif() -# This is here for cmake version 2.6 +# This is here for cmake version 2.6 if (NOT MPI_LIBRARY OR NOT MPI_EXTRA_LIBRARY) # Punt if MPI_LIBRARY or MPI_EXTRA_LIBRARY not found set(MPI_CXX_LIBRARIES "") else() if (NOT MPI_CXX_LIBRARIES) - set(MPI_CXX_LIBRARIES - ${MPI_LIBRARY} + set(MPI_CXX_LIBRARIES + ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY} ) endif() @@ -184,7 +184,7 @@ message(STATUS "MPI_EXTRA_LIBRARY: ${MPI_EXTRA_LIBRARY}") # Perkins found out that this was exactly the wrong thing to do: -# +# # set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) # set(CMAKE_CXX_COMPILE_FLAGS ${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}) # set(CMAKE_CXX_LINK_FLAGS ${CMAKE_CXX_LINK_FLAGS} ${MPI_LINK_FLAGS}) @@ -197,12 +197,12 @@ message(STATUS "MPI_EXTRA_LIBRARY: ${MPI_EXTRA_LIBRARY}") # real problems, but there are so many of them they obscure errors # that are causing identifiable problems -if (CMAKE_COMPILER_IS_GNUCXX) +if (CMAKE_COMPILER_IS_GNUCXX) add_definitions( -pedantic -Wno-write-strings - -Wno-long-long - -Wno-sign-compare + -Wno-long-long + -Wno-sign-compare -Wno-unused-variable -Wno-unused-but-set-variable -Wno-maybe-uninitialized @@ -250,7 +250,7 @@ endif() option(BOOST_BUILD_PYTHON "Include Boost::Python in the Boost build" OFF) # These are the Boost libraries required here - + set(BOOST_BUILD_LIBS "") list(APPEND BOOST_BUILD_LIBS mpi serialization random filesystem system) @@ -294,8 +294,8 @@ endif() if (NOT PETSC_DIR) message(FATAL_ERROR "PETSC_DIR is not set, do so with -D PETSC_DIR=...") endif() - -# Include petsc package path in pkg_config_path + +# Include petsc package path in pkg_config_path set(ENV{PKG_CONFIG_PATH} ${PETSC_DIR}/lib/pkgconfig:${PETSC_DIR}/${PETSC_ARCH}/lib/pkgconfig ) @@ -314,7 +314,7 @@ petscconf.h") endif() # checks - # define PETSc variables + # define PETSc variables include(CheckSymbolExists) check_symbol_exists(PETSC_HAVE_PARMETIS ${petscconf} PETSC_HAVE_PARMETIS) check_symbol_exists(PETSC_USE_REAL_DOUBLE ${petscconf} PETSC_USE_REAL_DOUBLE) @@ -336,7 +336,7 @@ else() message(STATUS "PETSC_LIBRARY_SINGLE: ${PETSC_LIBRARY_SINGLE}") - # checks + # checks if (NOT PETSC_HAVE_MPI) message(FATAL_ERROR "PETSc installation is not parallel (--with-mpi=1)") @@ -348,13 +348,13 @@ endif() # (--with-scalar-type=real). This is to determine what that # underlying type is. -if (PETSC_USE_REAL_DOUBLE) +if (PETSC_USE_REAL_DOUBLE) message(STATUS "PETSc installation is double precision (--with-precision=double) -- good") else() message(FATAL_ERROR "PETSc installation is not double precision (--with-precision=double)") endif() -if (PETSC_USE_COMPLEX) +if (PETSC_USE_COMPLEX) message (STATUS "PETSc installation uses complex type (--with-scalar-type=complex)") else() message (STATUS "PETSc installation uses real type (--with-scalar-type=real)") @@ -365,7 +365,7 @@ if (PETSC_CLANGUAGE_Cxx) message (STATUS "PETSc installation uses C++ (--with-clanguage=c++) -- we can work with that.") endif() -if (PETSC_HAVE_SUPERLU_DIST) +if (PETSC_HAVE_SUPERLU_DIST) set(GRIDPACK_MATSOLVER_PKG "superlu_dist") message(STATUS "PETSc parallel LU linear solver will be from SuperLU_dist") elseif (PETSC_HAVE_MUMPS) @@ -378,7 +378,7 @@ endif() # ------------------------------------------------------------- # ParMETIS -# +# # Only versions > 4.0 have been used # ------------------------------------------------------------- message(STATUS "Checking ParMETIS ...") @@ -424,7 +424,7 @@ if(DOXYGEN_FOUND) COMMENT "Generating API documentation with Doxygen" VERBATIM ) - + # The custom target "devdoc" generates "developer" # documentation. This has all the nitty gritty details and code @@ -545,7 +545,7 @@ add_subdirectory(lib) INCLUDE(InstallRequiredSystemLibraries) -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "An open source toolkit for developing power grid simulation applications for high performance computing architectures") set(CPACK_PACKAGE_DESCRIPTION "GridPACK(tm) is a software framework consisting of a set of modules designed to simplify the development of programs that model the power grid and run on parallel, high performance computing platforms. The modules are available as a library and consist of components for setting up and distributing power grid networks, support for modeling the behavior of individual buses and branches in the network, converting the network models to the corresponding algebraic equations, and parallel routines for manipulating and solving large algebraic systems. Additional modules support input and output as well as basic profiling and error management. ") set(CPACK_PACKAGE_VENDOR "") @@ -553,11 +553,11 @@ set(CPACK_PACKAGE_CONTACT "william.perkins@pnnl.gov") set(CPACK_PACKAGE_VERSION_MAJOR "${GridPACK_VERSION_MAJOR}") set(CPACK_PACKAGE_VERSION_MINOR "${GridPACK_VERSION_MINOR}") set(CPACK_PACKAGE_VERSION_PATCH "${GridPACK_VERSION_PATCH}") -set(CPACK_PACKAGE_VERSION +set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") -set(CPACK_PACKAGE_FILE_NAME +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${CPACK_PACKAGE_VERSION}") -set(CPACK_SOURCE_PACKAGE_FILE_NAME +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${CPACK_PACKAGE_VERSION}") # set(CPACK_PACKAGING_INSTALL_PREFIX "") @@ -571,7 +571,7 @@ set(CPACK_SET_DESTDIR true) set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional") set(CPACK_DEBIAN_PACKAGE_SECTION "science") set(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) -set(CPACK_DEBIAN_PACKAGE_DEPENDS +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libga-dev (>=5.3), libpetsc3.6.2-dev (>= 3.6.2), libparmetis-dev (>=4.0.3), libboost-mpi-dev (>=1.58.0), libboost-all-dev (>=1.58.0)" ) diff --git a/src/applications/dynamic_simulation_full_y/CMakeLists.txt b/src/applications/dynamic_simulation_full_y/CMakeLists.txt index b1117e357..4ea60b49f 100644 --- a/src/applications/dynamic_simulation_full_y/CMakeLists.txt +++ b/src/applications/dynamic_simulation_full_y/CMakeLists.txt @@ -33,7 +33,7 @@ set(target_libraries gridpack_parallel gridpack_block_parsers ${PETSC_LIBRARIES} - ${PARMETIS_LIBRARY} ${METIS_LIBRARY} + ${PARMETIS_LIBRARY} ${METIS_LIBRARY} ${Boost_LIBRARIES} ${GA_LIBRARIES} ${MPI_CXX_LIBRARIES}) @@ -124,60 +124,60 @@ gridpack_set_lu_solver( ) add_custom_target(dsf.x.input - - COMMAND ${CMAKE_COMMAND} -E copy + + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/IEEE_145b_classical_model.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/9b3g.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/9b3g.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/300bus_v23_no0imp_pslf.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/300bus_detail_model_cmpld_combine.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/bus3000_gen_no0imp_v23_pslf.raw ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/classical_model_3000bus.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/240busWECC_2018_PSS_mod.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/240busWECC_2018_PSS_fixedshunt.raw - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea.dyr ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/raw/kundur-twoarea_v33.raw - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy + COMMAND ${CMAKE_COMMAND} -E copy ${GRIDPACK_DATA_DIR}/dyr/kundur-twoarea_4renewable_mech.dyr ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw ${GRIDPACK_DATA_DIR}/dyr/IEEE_145b_classical_model.dyr @@ -212,7 +212,7 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.install.in ${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt @ONLY) -install(FILES +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.txt ${CMAKE_CURRENT_BINARY_DIR}/input_145.xml ${GRIDPACK_DATA_DIR}/raw/IEEE_145bus_v23_PSLF.raw @@ -233,7 +233,7 @@ install(FILES install(TARGETS dsf.x DESTINATION bin) install(TARGETS dsf2.x DESTINATION bin) - + # ------------------------------------------------------------- # run application as test # ------------------------------------------------------------- diff --git a/src/cmake-modules/GridPACK.cmake b/src/cmake-modules/GridPACK.cmake index 34ef15a05..cfedfe308 100644 --- a/src/cmake-modules/GridPACK.cmake +++ b/src/cmake-modules/GridPACK.cmake @@ -17,7 +17,7 @@ # This is used to specify a time out for GridPACK unit tests. It's 5 # seconds by default, but may need to be longer on some platforms. -if (NOT GRIDPACK_TEST_TIMEOUT) +if (NOT GRIDPACK_TEST_TIMEOUT) set (GRIDPACK_TEST_TIMEOUT 120 CACHE STRING "Time out for GridPACK unit tests.") endif () @@ -61,7 +61,7 @@ function(gridpack_add_serial_unit_test test_name test_target) set(the_test_name "${test_name}_serial") add_test("${the_test_name}" "${test_target}") set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES PASS_REGULAR_EXPRESSION "No errors detected" FAIL_REGULAR_EXPRESSION "failure detected" TIMEOUT ${GRIDPACK_TEST_TIMEOUT} @@ -80,7 +80,7 @@ function(gridpack_add_serial_run_test test_name test_target test_input) set(the_test_name "${test_name}_serial") add_test("${the_test_name}" "${test_target}" ${test_input}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") @@ -95,13 +95,13 @@ function(gridpack_add_parallel_unit_test test_name test_target) add_test("${the_test_name}" ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${test_target} ${MPIEXEC_POSTFLAGS}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES PASS_REGULAR_EXPRESSION "No errors detected" FAIL_REGULAR_EXPRESSION "failure detected" TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") - else() + else() message(FATAL_ERROR "gridpack_add_parallel_unit_test: target argument not target") endif() endfunction(gridpack_add_parallel_unit_test) @@ -119,7 +119,7 @@ function(gridpack_add_parallel_run_test test_name test_target test_input) add_test("${the_test_name}" ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${test_target} ${MPIEXEC_POSTFLAGS} ${test_input}) set_tests_properties("${the_test_name}" - PROPERTIES + PROPERTIES TIMEOUT ${GRIDPACK_TEST_TIMEOUT} ) set_tests_ldpath("${the_test_name}") @@ -140,7 +140,7 @@ function(gridpack_add_unit_test test_name test_target) if (NOT USE_PROGRESS_RANKS) gridpack_add_serial_unit_test("${test_name}" ${test_target}) endif() - if (MPIEXEC) + if (MPIEXEC) gridpack_add_parallel_unit_test("${test_name}" ${test_target}) endif () endfunction(gridpack_add_unit_test) @@ -157,7 +157,7 @@ function(gridpack_add_run_test test_name test_target test_input) if (NOT USE_PROGRESS_RANKS) gridpack_add_serial_run_test("${test_name}" ${test_target} "${test_input}") endif() - if (MPIEXEC) + if (MPIEXEC) gridpack_add_parallel_run_test("${test_name}" ${test_target} "${test_input}") endif () endfunction(gridpack_add_run_test) diff --git a/src/example_configuration.sh b/src/example_configuration.sh index 3ba337e70..52d459598 100755 --- a/src/example_configuration.sh +++ b/src/example_configuration.sh @@ -121,7 +121,7 @@ elif [ $host == "we32673" ]; then CC=/opt/local/bin/clang export CC CXX=/opt/local/bin/clang++ - export CC CXX + export CC CXX prefix="/Users/d3g096/Projects/GridPACK" pdir="$prefix/petsc.gitlab" @@ -132,7 +132,7 @@ elif [ $host == "we32673" ]; then else parch="darwin-mpich-clang-complex-opt-3.16-static" fi - + cmake $options \ --graphviz=GridPACK.dot \ -D GA_DIR:STRING="$prefix/gridpack-install" \ @@ -204,8 +204,8 @@ elif [ $host == "tlaloc" ]; then # Ubuntu 20 with as many system packages as possible: GNU # compilers 9.4.0, OpenMPI 4.0.3, Boost 1.71.0, PETSc 3.12, - # ParMETIS 4.0.3. - + # ParMETIS 4.0.3. + CC=gcc CXX=g++ CFLAGS=-pthread @@ -232,21 +232,27 @@ elif [ $host == "tlaloc" ]; then # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" - cmake -Wdev --debug-trycompile \ - - # Custom built 3.12.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.12.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.12" \ + # Custom built 3.19, complex: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ - # Custom built 3.10.5, real: - # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.10.5" \ - # -D PETSC_ARCH:STRING="ubuntu-real-shared-3.10" \ + # Custom built 3.19, real: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-real-shared" \ # -D USE_OLD_PETSC:BOOL=OFF \ + # Custom built 3.19, complex, w/o superlu_dist: + # -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.19.4" \ + # -D PETSC_ARCH:STRING="ubuntu-complex-shared-mumps" \ + # -D USE_OLD_PETSC:BOOL=OFF \ - prefix="$HOME/Projects/GridPakLDRD/gridpack-install" + if [ -z "$GRIDPACK_DIR" ]; then + prefix="$HOME/Projects/ExaLearn/gridpack-install" + else + prefix="$GRIDPACK_DIR" + fi + cmake -Wdev --debug-trycompile \ --graphviz=GridPACK.dot \ -D PETSC_DIR:STRING="/home/d3g096/Projects/GridPakLDRD/petsc-3.14.6" \ @@ -318,6 +324,6 @@ else echo "Unknown host: $host" exit 2 - + fi diff --git a/src/lib/GridPACK.cmake.in b/src/lib/GridPACK.cmake.in index 856faf52e..5e04e1617 100644 --- a/src/lib/GridPACK.cmake.in +++ b/src/lib/GridPACK.cmake.in @@ -20,7 +20,7 @@ function(gridpack_setup) endif() set(gp_include "${GRIDPACK_DIR}/include") - + set(GRIDPACK_LIB_DIR "${GRIDPACK_DIR}/lib") set(GRIDPACK_HAVE_PETSC @PETSC_FOUND@) @@ -80,7 +80,7 @@ function(gridpack_setup) ) message(STATUS "GRIDPACK_HAVE_GOSS: ${GRIDPACK_HAVE_GOSS}") - if (GRIDPACK_HAVE_GOSS) + if (GRIDPACK_HAVE_GOSS) message(STATUS "Adding GRIDPACK_GOSS library") find_library(GRIDPACK_GOSS_LIBRARY NAMES gridpack_goss @@ -162,12 +162,12 @@ function(gridpack_setup) ${GRIDPACK_PARALLEL_LIBRARY} ) - if (GRIDPACK_HAVE_PARMETIS) + if (GRIDPACK_HAVE_PARMETIS) list(APPEND gp_include @PARMETIS_INCLUDE_DIR@ ) list(APPEND gp_libs - @PARMETIS_LIBRARY@ + @PARMETIS_LIBRARY@ @METIS_LIBRARY@ ) endif() @@ -216,16 +216,16 @@ function(gridpack_setup) list(APPEND gp_libs @GLPK_LIBRARY@) endif() - list(APPEND gp_include + list(APPEND gp_include @Boost_INCLUDE_DIR@ @MPI_INCLUDE_PATH@ ) - + list(APPEND gp_libs @Boost_LIBRARIES@ @MPI_CXX_LIBRARIES@ ) - + set(GRIDPACK_INCLUDE_DIRS ${gp_include} CACHE STRING "Include directories for GridPACK" From 41bc8cc506e8360fdaf7e89a6d20c34e179803fb Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 29 Dec 2023 12:09:04 -0600 Subject: [PATCH 069/125] Remove attempted variable substitution --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3e38ecb7..c48156fb1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,12 +16,15 @@ variables: parallel: matrix: - - BASE_IMAGE: rockylinux:9 - - BASE_IMAGE: ubuntu:22.04 + + - BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" + - BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" variables: - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE%%:*}-${CI_COMMIT_REF_NAME}" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-${CI_COMMIT_REF_NAME}" # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -40,7 +43,7 @@ build-container: - /kaniko/executor --context "${CI_PROJECT_DIR}" - --build-arg BASE_IMAGE=${BASE_IMAGE} + --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG} --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --dockerfile "${CI_PROJECT_DIR}/dockerfile" From c985f451d1d90cf3c47f2d1f49c72a89fa35ad4d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:37:39 -0600 Subject: [PATCH 070/125] Set conditions for when container is built --- .gitlab-ci.yml | 60 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c48156fb1..a155f2640 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,30 @@ variables: variables: - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-${CI_COMMIT_REF_NAME}" + IMAGE_TAG: ${BASE_IMAGE}-gridpack-env + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} + + +check-container-definition-changed: + + extends: .multi-distro + + image: alpine + + script: + + - touch container-definition-changed + + artifacts: + + paths: + + - container-definition-changed + + rules: + + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile" ] # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -39,22 +62,34 @@ build-container: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] + needs: + + - job: check-container-definition-changed + optional: true + script: + # build image if definition was changed or the tag does not exist in the registry + - | + tag_exists() { + curl \ + --silent \ + --show-error \ + --location \ + --fail \ + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG\" \ + 1>$null 2>$null + } + - test -f container-definition-changed || ! tag_exists || exit 0 - /kaniko/executor --context "${CI_PROJECT_DIR}" - --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG} - --build-arg http_proxy=$http_proxy - --build-arg https_proxy=$https_proxy + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" - rules: - - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile" ] - - if: $CI_PIPELINE_SOURCE == "tag" - build-gridpack: @@ -66,10 +101,7 @@ build-gridpack: MAKE_JOBS: "2" - needs: - - - job: build-container - optional: true + needs: [ build-container ] script: From be884de76c4eaf6d1499e3e0e278283097f9031d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:42:02 -0600 Subject: [PATCH 071/125] Fix quoting --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a155f2640..bdd62ca71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,7 +78,7 @@ build-container: --location \ --fail \ --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG\" \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ 1>$null 2>$null } - test -f container-definition-changed || ! tag_exists || exit 0 From 46969d6c2cde4c3ca0428efccd3972cc675f7263 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 09:53:22 -0600 Subject: [PATCH 072/125] Make script available to setup env --- dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dockerfile b/dockerfile index b40b5f5f6..24d999ec8 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,6 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -RUN install_environment_packages.sh - ENV GP_EXT_DEPS=/gridpack-dependencies ENV BOOST_VERSION "1.78.0" ENV GA_VERSION "5.8" @@ -13,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN ./install_gridpack_deps.sh && rm *.sh +RUN ./install_environment_packages.sh && ./install_gridpack_deps.sh && rm *.sh From b83499ed8842d141f035022538ac456037ad8795 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 10:19:09 -0600 Subject: [PATCH 073/125] Make scripts executable for docker image build --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd62ca71..82944c5ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,8 +30,6 @@ variables: check-container-definition-changed: - extends: .multi-distro - image: alpine script: @@ -82,6 +80,7 @@ build-container: 1>$null 2>$null } - test -f container-definition-changed || ! tag_exists || exit 0 + - chmod +x *.sh - /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" From d7bf55758fc6d4a7a44a60850826fe0d0f8d760f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 10:57:23 -0600 Subject: [PATCH 074/125] Save logs from install_gridpack_deps --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82944c5ae..7b001c15c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,6 +89,10 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" + artifacts: + + paths: [ log/ ] + build-gridpack: From 71099a31e185fe368c642bc7528817fcf04752fa Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 14:53:13 -0600 Subject: [PATCH 075/125] Fix artifact path for logs --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b001c15c..296bc6e2d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,7 +91,7 @@ build-container: artifacts: - paths: [ log/ ] + paths: [ /gridpack-dependencies/log/ ] build-gridpack: From 401494ae7c9f320e75fa38aabbd5b8822d2cd148 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 2 Jan 2024 16:21:45 -0600 Subject: [PATCH 076/125] Test - remove parallel config --- .gitlab-ci.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 296bc6e2d..6523c7562 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,17 +13,10 @@ variables: .multi-distro: - parallel: - - matrix: - - - BASE_IMAGE: rockylinux - BASE_IMAGE_TAG: "9" - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - variables: + BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" IMAGE_TAG: ${BASE_IMAGE}-gridpack-env IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} From 6bd83368814515049773b59fff2e5aa508e8f346 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 09:59:36 -0600 Subject: [PATCH 077/125] Link logs to project folder - artifacts can only be saved from there per gitlab docs --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6523c7562..2c20a8edd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,6 +60,7 @@ build-container: script: + - ln -s /gridpack-dependencies/log/ ${CI_PROJECT_DIR}/log # build image if definition was changed or the tag does not exist in the registry - | tag_exists() { @@ -84,7 +85,7 @@ build-container: artifacts: - paths: [ /gridpack-dependencies/log/ ] + paths: [ log/ ] build-gridpack: From f9a76ec86a30c2b9552dcb9148d29bcff9d9e750 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 10:34:02 -0600 Subject: [PATCH 078/125] =?UTF-8?q?Send=20logs=20to=20back=20console=20?= =?UTF-8?q?=F0=9F=98=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 5 ----- install_gridpack_deps.sh | 21 +++++++++------------ 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c20a8edd..e58236720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,6 @@ build-container: script: - - ln -s /gridpack-dependencies/log/ ${CI_PROJECT_DIR}/log # build image if definition was changed or the tag does not exist in the registry - | tag_exists() { @@ -83,10 +82,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" - artifacts: - - paths: [ log/ ] - build-gridpack: diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 98a6bd29e..a9a1950c4 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -35,17 +35,16 @@ function install_boost { echo "Bootstrapping Boost" ./bootstrap.sh \ --prefix=install_for_gridpack \ - --with-libraries=mpi,serialization,random,filesystem,system \ - >../log/boost_bootstrap.log 2>&1 + --with-libraries=mpi,serialization,random,filesystem,system echo 'using mpi ;' >>project-config.jam # build echo "Building Boost" - ./b2 -a -d+2 link=shared stage >../log/boost_build.log 2>&1 + ./b2 -a -d+2 link=shared stage # install echo "Installing Boost" - ./b2 -a -d+2 link=shared install >../log/boost_install.log 2>&1 + ./b2 -a -d+2 link=shared install popd || exit @@ -85,12 +84,11 @@ function install_ga { --enable-cxx \ --enable-i4 \ --prefix="${PWD}/install_for_gridpack" \ - --enable-shared \ - >../log/ga_configure.log 2>&1 + --enable-shared # install echo "Installing Global Arrays" - make -j "$(nproc)" install >../log/ga_install.log 2>&1 + make -j "$(nproc)" install popd || exit @@ -128,20 +126,19 @@ function install_petsc { --prefix="${PWD}"/install_for_gridpack \ --scalar-type=complex \ --with-shared-libraries=1 \ - --download-f2cblaslapack=1 \ - >../log/petsc_configure.log 2>&1 + --download-f2cblaslapack=1 # build echo "Building PETSc" - make >../log/petsc_build.log 2>&1 + make # install echo "Installing PETSc" - make install >../log/petsc_install.log 2>&1 + make install # check echo "Checking PETSc" - make check >../log/petsc_check.log 2>&1 + make check popd || exit From 7d842a30bc5dcc48d3bb2cf276b9128585000f1d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 13:23:51 -0600 Subject: [PATCH 079/125] Redirect output to files --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e58236720..2c82b27be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,7 +80,11 @@ build-container: --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" + --destination "${IMAGE_PATH}" 2> build_stderr.log | tee build_stdout.log + + artifacts: + + paths: [ build_stderr.log, build_stdout.log ] build-gridpack: From 2910f3bb80476a14ec3536e075f19a2d2eab0962 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 13:25:40 -0600 Subject: [PATCH 080/125] Build PETSc with MUMPS instead of SuperLU_dist --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index a9a1950c4..46f4e9970 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-superlu_dist \ + --download-mumps \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 186b938b7fe73dddfbe7d945d1c4ca0f58b5d677 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 3 Jan 2024 14:23:25 -0600 Subject: [PATCH 081/125] Redirect both streams from kaniko exe to file Still not getting an artifact in gitlab... not sure if it's bc I'm overflowing the console --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c82b27be..70a7ba9b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,11 +80,11 @@ build-container: --build-arg "http_proxy=$http_proxy" --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" 2> build_stderr.log | tee build_stdout.log + --destination "${IMAGE_PATH}" > build.log 2>&1 artifacts: - paths: [ build_stderr.log, build_stdout.log ] + paths: [ build.log ] build-gridpack: From 65965a59d48dd9ea0e45170ba9678ba295ae9122 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 12:12:07 -0600 Subject: [PATCH 082/125] Move the container registry check to its own job with curl actually installed --- .gitlab-ci.yml | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70a7ba9b5..512e209cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,9 +31,7 @@ check-container-definition-changed: artifacts: - paths: - - - container-definition-changed + untracked: true rules: @@ -41,6 +39,28 @@ check-container-definition-changed: changes: [ "dockerfile" ] +check-container-tag-exists: + + image: alpine + + script: + + - apk add --no-cache curl + - > + curl \ + --silent \ + --show-error \ + --location \ + --fail \ + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ + 1>$null 2>$null || touch container-tag-not-found + + artifacts: + + untracked: true + + # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: @@ -57,22 +77,12 @@ build-container: - job: check-container-definition-changed optional: true + - job: check-container-tag-exists script: # build image if definition was changed or the tag does not exist in the registry - - | - tag_exists() { - curl \ - --silent \ - --show-error \ - --location \ - --fail \ - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ - 1>$null 2>$null - } - - test -f container-definition-changed || ! tag_exists || exit 0 + - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - /kaniko/executor --context "${CI_PROJECT_DIR}" From 6ca911f9fdfc88938314bf9e8c22ae067ace085b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:26:02 -0600 Subject: [PATCH 083/125] Use block scalar to avoid issues with special chars --- .gitlab-ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 512e209cc..1e80a1f2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,13 +84,14 @@ build-container: # build image if definition was changed or the tag does not exist in the registry - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - - /kaniko/executor - --context "${CI_PROJECT_DIR}" - --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" > build.log 2>&1 + - > + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --destination "${IMAGE_PATH}" > build.log 2>&1 artifacts: From 25ddd7966c91cf040a9703669dc6ddc116b07f20 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:27:05 -0600 Subject: [PATCH 084/125] Always save untracked files --- .gitlab-ci.yml | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e80a1f2a..2c617e2b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,11 @@ default: tags: [ basic, gridpack, ikp, k8s ] + artifacts: + + when: always + untracked: true + variables: @@ -29,10 +34,6 @@ check-container-definition-changed: - touch container-definition-changed - artifacts: - - untracked: true - rules: - if: $CI_PIPELINE_SOURCE == "push" @@ -56,10 +57,6 @@ check-container-tag-exists: "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ 1>$null 2>$null || touch container-tag-not-found - artifacts: - - untracked: true - # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy build-container: @@ -93,10 +90,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - artifacts: - - paths: [ build.log ] - build-gridpack: @@ -114,10 +107,6 @@ build-gridpack: - ./install_gridpack.sh - artifacts: - - untracked: true - test-gridpack: @@ -132,11 +121,3 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' - - artifacts: - - paths: - - - src/build/Testing/Temporary - - when: always From fe636033ae5692668310cf160b2955a806f2f7c4 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 13:39:54 -0600 Subject: [PATCH 085/125] Fix block scalar Lines with indentation are not folded! https://yaml-multiline.info/ --- .gitlab-ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c617e2b2..7d33e9186 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,14 +48,14 @@ check-container-tag-exists: - apk add --no-cache curl - > - curl \ - --silent \ - --show-error \ - --location \ - --fail \ - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " \ - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" \ - 1>$null 2>$null || touch container-tag-not-found + curl + --silent + --show-error + --location + --fail + --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" + 1>$null 2>$null || touch container-tag-not-found # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -83,12 +83,12 @@ build-container: - chmod +x *.sh - > /kaniko/executor - --context "${CI_PROJECT_DIR}" - --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" - --dockerfile "${CI_PROJECT_DIR}/dockerfile" - --destination "${IMAGE_PATH}" > build.log 2>&1 + --context "${CI_PROJECT_DIR}" + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --destination "${IMAGE_PATH}" > build.log 2>&1 build-gridpack: From 9e2637071f4b875136f309ea5c1eb1f40507c541 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:06:23 -0600 Subject: [PATCH 086/125] Swap out mpms for superlu for now PETSc config fails due to options and I need to figure out a way to get the more detailed log from outside the project folder (as a gitlab artifact). --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 46f4e9970..a9a1950c4 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-mumps \ + --download-superlu_dist \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 70deb07bd83a903b3ac30bd03b30ff748f7750cc Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:25:29 -0600 Subject: [PATCH 087/125] Pull back on untracked artifacts where it makes sense --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d33e9186..778fe8728 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,10 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 + artifacts: + + paths: [ build.log ] + build-gridpack: @@ -121,3 +125,7 @@ test-gridpack: - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' + + artifacts: + + paths: [ src/build/Testing/Temporary ] From 8c916009316763f42708abedbbd556760a551ad6 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:34:03 -0600 Subject: [PATCH 088/125] Revert "Swap out mpms for superlu for now" This reverts commit 9e2637071f4b875136f309ea5c1eb1f40507c541. --- install_gridpack_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index a9a1950c4..46f4e9970 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -117,7 +117,7 @@ function install_petsc { # install echo "Configuring PETSc" ./configure \ - --download-superlu_dist \ + --download-mumps \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 9253abf067157f1ae364e8d11a606917dfc96d9b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 14:35:00 -0600 Subject: [PATCH 089/125] Add scalapack per error message --- install_gridpack_deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 46f4e9970..640613561 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -118,6 +118,7 @@ function install_petsc { echo "Configuring PETSc" ./configure \ --download-mumps \ + --download-scalapack \ --download-metis \ --download-parmetis \ --download-suitesparse \ From 37ad180bf21a908914a61b84d23975c7ce12013c Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 15:18:45 -0600 Subject: [PATCH 090/125] Remove the artifact path spec for build container For some reason this wasn't saved again... --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 778fe8728..8d558ea7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,10 +90,6 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - artifacts: - - paths: [ build.log ] - build-gridpack: From 574be1cc49ec51df41882a4bec7bb4c411562b65 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 16:13:46 -0600 Subject: [PATCH 091/125] Pull container repo id into var --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d558ea7a..5296a0a25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,10 @@ check-container-tag-exists: image: alpine + variables: + + CONTAINER_REPO_ID: "294" + script: - apk add --no-cache curl @@ -54,7 +58,7 @@ check-container-tag-exists: --location --fail --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/294/tags/$IMAGE_TAG" + "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$CONTAINER_REPO_ID/tags/$IMAGE_TAG" 1>$null 2>$null || touch container-tag-not-found From cc7ecaec0bf5bd5265be109d5749e1a792f5ccf0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 5 Jan 2024 16:14:13 -0600 Subject: [PATCH 092/125] Try to see log files I need when build fails --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 24d999ec8..38bc2ff35 100644 --- a/dockerfile +++ b/dockerfile @@ -11,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN ./install_environment_packages.sh && ./install_gridpack_deps.sh && rm *.sh +RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || find ${GP_EXT_DEPS} -type f -regex ".*log.*" && rm *.sh From 1ae11b161f47d9b901c5d735f5b1f943e502df0e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 18:28:55 -0600 Subject: [PATCH 093/125] Set the proxy var for kaniko --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5296a0a25..89482c8c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" + PROXY: "http://proxy01.pnl.gov:3128" .multi-distro: @@ -89,8 +90,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$http_proxy" - --build-arg "https_proxy=$https_proxy" + --build-arg "http_proxy=$PROXY" + --build-arg "https_proxy=$PROXY" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 From 317ddc4c16832171242e71e1307c683414ed7f93 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 19:23:35 -0600 Subject: [PATCH 094/125] Pass proxy to all jobs --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89482c8c1..b8715374f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,8 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" - PROXY: "http://proxy01.pnl.gov:3128" + HTTP_PROXY: "http://proxy01.pnl.gov:3128" + HTTPS_PROXY: HTTP_PROXY .multi-distro: @@ -90,8 +91,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$PROXY" - --build-arg "https_proxy=$PROXY" + --build-arg "http_proxy=$HTTP_PROXY" + --build-arg "https_proxy=$HTTPS_PROXY" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 From bb44759a98c7bfbb2f944ed7e995f78f4c540eed Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 21:07:03 -0600 Subject: [PATCH 095/125] Try lowercase --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8715374f..31b770275 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,8 @@ variables: KUBERNETES_MEMORY_LIMIT: "8Gi" HTTP_PROXY: "http://proxy01.pnl.gov:3128" HTTPS_PROXY: HTTP_PROXY + http_proxy: HTTP_PROXY + https_proxy: HTTP_PROXY .multi-distro: From cae4119e83650255bcc8ca4396e7dbbae17de8e0 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 9 Jan 2024 21:11:02 -0600 Subject: [PATCH 096/125] Mark as variables --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31b770275..ba4ab8533 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,9 +15,9 @@ variables: KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" HTTP_PROXY: "http://proxy01.pnl.gov:3128" - HTTPS_PROXY: HTTP_PROXY - http_proxy: HTTP_PROXY - https_proxy: HTTP_PROXY + HTTPS_PROXY: ${HTTP_PROXY} + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTP_PROXY} .multi-distro: From 6a576eee10ad926792d9c81aa79e701a17f96a26 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Fri, 19 Jan 2024 16:04:37 -0600 Subject: [PATCH 097/125] Remove petsc config option --- install_gridpack_deps.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 640613561..95e3a451d 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -126,8 +126,7 @@ function install_petsc { --download-cmake \ --prefix="${PWD}"/install_for_gridpack \ --scalar-type=complex \ - --with-shared-libraries=1 \ - --download-f2cblaslapack=1 + --with-shared-libraries=1 # build echo "Building PETSc" From d0113b420864ffd8bedb34d084d0a553d8dd96ba Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:51:25 -0600 Subject: [PATCH 098/125] Remove unused log folder --- install_gridpack_deps.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 95e3a451d..f0e0fa310 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -148,8 +148,6 @@ function install_petsc { echo "Installing GridPACK dependencies" date -mkdir -p log - install_boost "${BOOST_VERSION:?}" install_ga "${GA_VERSION:?}" install_petsc "${PETSC_VERSION:?}" From e21f58f193266bb8d5c4d4e7336ed5cb27a5288b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:52:25 -0600 Subject: [PATCH 099/125] Try to parallelize the pipeline again Do everything for ubuntu and rockylinux --- .gitlab-ci.yml | 62 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba4ab8533..853e022d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,48 +22,16 @@ variables: .multi-distro: - variables: - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - IMAGE_TAG: ${BASE_IMAGE}-gridpack-env - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${IMAGE_TAG} - - -check-container-definition-changed: - - image: alpine - - script: + parallel: - - touch container-definition-changed + matrix: - rules: - - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile" ] - - -check-container-tag-exists: - - image: alpine - - variables: - - CONTAINER_REPO_ID: "294" - - script: - - - apk add --no-cache curl - - > - curl - --silent - --show-error - --location - --fail - --header "PRIVATE-TOKEN: $CI_JOB_TOKEN " - "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$CONTAINER_REPO_ID/tags/$IMAGE_TAG" - 1>$null 2>$null || touch container-tag-not-found + - BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env + - BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy @@ -78,16 +46,8 @@ build-container: name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - needs: - - - job: check-container-definition-changed - optional: true - - job: check-container-tag-exists - script: - # build image if definition was changed or the tag does not exist in the registry - - test -f container-definition-changed || test -f container-tag-not-found || exit 0 - chmod +x *.sh - > /kaniko/executor @@ -98,6 +58,12 @@ build-container: --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 + rules: + + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + - when: manual + build-gridpack: From 2f9ae934c2223193d14c047641e8694c0acfb480 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 15:57:35 -0600 Subject: [PATCH 100/125] Revert troubleshooting command --- dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfile b/dockerfile index 38bc2ff35..ad7a5ee14 100644 --- a/dockerfile +++ b/dockerfile @@ -11,4 +11,4 @@ WORKDIR ${GP_EXT_DEPS} COPY *.sh . -RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || find ${GP_EXT_DEPS} -type f -regex ".*log.*" && rm *.sh +RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || rm *.sh From 8f1ccb4a433df6a03fb2e3bde68cb3467c2c32a1 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 16:24:48 -0600 Subject: [PATCH 101/125] Separate scripts to improve caching --- dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dockerfile b/dockerfile index ad7a5ee14..2ae8c0cd7 100644 --- a/dockerfile +++ b/dockerfile @@ -9,6 +9,8 @@ ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS} WORKDIR ${GP_EXT_DEPS} -COPY *.sh . +COPY install_environment_packages.sh . +RUN ./install_environment_packages.sh && rm *.sh -RUN (./install_environment_packages.sh && ./install_gridpack_deps.sh) || rm *.sh +COPY install_gridpack_deps.sh . +RUN ./install_gridpack_deps.sh && rm *.sh From 9af3251b5148bebf4ee53586c85d0828e1fede7a Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Sat, 20 Jan 2024 16:34:27 -0600 Subject: [PATCH 102/125] Install mpi4py dependency --- install_gridpack.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_gridpack.sh b/install_gridpack.sh index 688832d4e..e2a3d44e6 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -91,6 +91,9 @@ function install_gridpack_python { # export GRIDPACK_DIR export GRIDPACK_DIR="${gridpack_install_dir}" + # install dependencies + pip install mpi4py --user + # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build From da312308c9516d2fb450cff7288d583e9af390de Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:08:41 -0600 Subject: [PATCH 103/125] Set proxy build args in container build env --- .gitlab-ci.yml | 10 ++++------ dockerfile | 5 +++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 853e022d6..756de6eb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,10 +14,8 @@ variables: KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" KUBERNETES_MEMORY_LIMIT: "8Gi" - HTTP_PROXY: "http://proxy01.pnl.gov:3128" - HTTPS_PROXY: ${HTTP_PROXY} - http_proxy: ${HTTP_PROXY} - https_proxy: ${HTTP_PROXY} + http_proxy: "http://proxy01.pnl.gov:3128" + https_proxy: ${http_proxy} .multi-distro: @@ -53,8 +51,8 @@ build-container: /kaniko/executor --context "${CI_PROJECT_DIR}" --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" - --build-arg "http_proxy=$HTTP_PROXY" - --build-arg "https_proxy=$HTTPS_PROXY" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 diff --git a/dockerfile b/dockerfile index 2ae8c0cd7..92bb67862 100644 --- a/dockerfile +++ b/dockerfile @@ -7,6 +7,11 @@ ENV GA_VERSION "5.8" ENV PETSC_VERSION "3.16.4" ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +ARG http_proxy +ARG https_proxy +ENV http_proxy=${http_proxy} +ENV https_proxy=${https_proxy} + WORKDIR ${GP_EXT_DEPS} COPY install_environment_packages.sh . From 109ee8972fa11c24fae05ce4b41be8d3e6443a5f Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:09:50 -0600 Subject: [PATCH 104/125] Get mpi4py from system repos instead of through pip --- install_environment_packages.sh | 4 ++-- install_gridpack.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 9109cc8d7..9b06d630e 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -15,12 +15,12 @@ debian | ubuntu) apt update && apt upgrade -y && apt install -y \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; fedora | rhel | centos | rocky) dnf update -y && dnf install -y \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi ;; *) echo "$distribution not supported" diff --git a/install_gridpack.sh b/install_gridpack.sh index e2a3d44e6..688832d4e 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -91,9 +91,6 @@ function install_gridpack_python { # export GRIDPACK_DIR export GRIDPACK_DIR="${gridpack_install_dir}" - # install dependencies - pip install mpi4py --user - # build echo "Building GridPACK python wrapper" ${python_exe} setup.py build From 65cd8639882ea9d16db5b75e62aee3ee684da412 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 12:10:08 -0600 Subject: [PATCH 105/125] Use common check for distro between scripts --- install_gridpack.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 688832d4e..0b0e17140 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -74,13 +74,16 @@ function install_gridpack_python { pushd python || exit # set an env var if we are running on RHEL - os_id=$( + distribution=$( + # shellcheck source=/dev/null source /etc/os-release echo "$ID" ) - if [[ $os_id == "rhel" ]] || [[ $os_id == "centos" ]]; then - export RHEL_OPENMPI_HACK=yes - fi + case $distribution in + fedora | rhel | centos | rocky) + export RHEL_OPENMPI_HACK=yes + ;; + esac # set python executable path python_exe=$(which python || which python3) From 31d1fb274d43a75264f408cc3fe853f9a827cc6e Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Mon, 22 Jan 2024 16:47:56 -0600 Subject: [PATCH 106/125] Manually define job parallelization and adjust whitespace --- .gitlab-ci.yml | 79 +++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 756de6eb4..6074c8211 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,11 @@ default: - tags: [ basic, gridpack, ikp, k8s ] - artifacts: - when: always untracked: true variables: - KUBERNETES_CPU_REQUEST: "1" KUBERNETES_CPU_LIMIT: "2" KUBERNETES_MEMORY_REQUEST: "4Gi" @@ -18,34 +14,15 @@ variables: https_proxy: ${http_proxy} -.multi-distro: - - parallel: - - matrix: - - - BASE_IMAGE: ubuntu - BASE_IMAGE_TAG: "22.04" - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env - - BASE_IMAGE: rockylinux - BASE_IMAGE_TAG: "9" - IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env - - # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy -build-container: - - extends: .multi-distro - +.build-container: timeout: 5 hours - image: - name: gcr.io/kaniko-project/executor:v1.9.0-debug entrypoint: [""] - + variables: + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - chmod +x *.sh - > /kaniko/executor @@ -55,45 +32,61 @@ build-container: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] - when: manual -build-gridpack: - - extends: .multi-distro +build-container:ubuntu: + extends: .build-container + variables: + BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" - image: "${IMAGE_PATH}" +build-container:rockylinux: + extends: .build-container variables: + BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" - MAKE_JOBS: "2" - - needs: [ build-container ] +.build-gridpack: + variables: + MAKE_JOBS: "2" script: - - ./install_gridpack.sh -test-gridpack: +build-gridpack:ubuntu: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: [ build-container:ubuntu ] - extends: .multi-distro - image: ${IMAGE_PATH} +build-gridpack:rockylinux: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: [ build-container:rockylinux ] - needs: [ build-gridpack ] +.test-gridpack: script: - - useradd gridpack - chown -R gridpack:gridpack . - su gridpack -c 'ctest --test-dir src/build --output-on-failure' - artifacts: - paths: [ src/build/Testing/Temporary ] + + +test-gridpack:ubuntu: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: [ build-gridpack:ubuntu ] + + +test-gridpack:rockylinux: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: [ build-gridpack:rockylinux ] From 3fc6c43b6cb0c36e26248e7070763fc4cebd71cb Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:23:26 -0600 Subject: [PATCH 107/125] Try adding docker config and cert to kaniko container Also disable the ubuntu jobs for now --- .gitlab-ci.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6074c8211..d6226bc93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,21 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: + - mkdir -p /kaniko/.docker + - | + { + "proxies": { + "default": { + "httpProxy": "${http_proxy}", + "httpsProxy": "${https_proxy}" + } + } + } >> /kaniko/.docker/config.json + + if [ -f /etc/gitlab-runner/certs/ca.crt ]; then + cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt + fi + - chmod +x *.sh - > /kaniko/executor @@ -38,7 +53,8 @@ variables: - when: manual -build-container:ubuntu: +# disabled for troubleshooting +.build-container:ubuntu: extends: .build-container variables: BASE_IMAGE: ubuntu @@ -59,7 +75,8 @@ build-container:rockylinux: - ./install_gridpack.sh -build-gridpack:ubuntu: +# disabled for troubleshooting +.build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-container:ubuntu ] @@ -80,7 +97,8 @@ build-gridpack:rockylinux: paths: [ src/build/Testing/Temporary ] -test-gridpack:ubuntu: +# disabled for troubleshooting +. test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-gridpack:ubuntu ] From 6eede55a19fdd0191155dc127182945ab73f3220 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:24:27 -0600 Subject: [PATCH 108/125] Make the build container job an optional dep --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6226bc93..f0e03fb84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,13 +79,17 @@ build-container:rockylinux: .build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env - needs: [ build-container:ubuntu ] + needs: + - job: build-container:ubuntu + optional: true build-gridpack:rockylinux: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env - needs: [ build-container:rockylinux ] + needs: + - job: build-container:rockylinux + optional: true .test-gridpack: From 8464d3142ae0f50b00a04e330a49957581f21322 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:33:38 -0600 Subject: [PATCH 109/125] Add verbose flag --- install_environment_packages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 9b06d630e..162771989 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -18,8 +18,8 @@ debian | ubuntu) wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; fedora | rhel | centos | rocky) - dnf update -y && - dnf install -y \ + dnf upgrade --assumeyes --verbose && + dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi ;; *) From c54637c057ec37f9f8b21e93db9b484e6c326d45 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:37:07 -0600 Subject: [PATCH 110/125] Remove manual trigger from container build job This always adds the job to the pipeline and blocks - ignoring the intent of it being an optional dependency downstream. I suppose if we want to rebuild the container, you could always go back to the last pipeline where it ran and rerun. This should be the last place that any files relevant to the container build changed. --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0e03fb84..4d49461ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,6 @@ variables: rules: - if: $CI_PIPELINE_SOURCE == "push" changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] - - when: manual # disabled for troubleshooting From d3222bb0eb706c99bad1c0ae62d975a26b711b9b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 10:40:25 -0600 Subject: [PATCH 111/125] Use a heredoc to populate docker config --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d49461ae..235de802e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ variables: script: - mkdir -p /kaniko/.docker - | + cat << EOF > /kaniko/.docker/config.json { "proxies": { "default": { @@ -32,7 +33,8 @@ variables: "httpsProxy": "${https_proxy}" } } - } >> /kaniko/.docker/config.json + } + EOF if [ -f /etc/gitlab-runner/certs/ca.crt ]; then cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt From 98877aeb9520c92956b811b9e3e4b07aeecde951 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 11:06:36 -0600 Subject: [PATCH 112/125] Dump environment variables --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 235de802e..e666a6149 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,7 @@ variables: fi - chmod +x *.sh + - env - > /kaniko/executor --context "${CI_PROJECT_DIR}" From 78f5ace1af96014051fb036ff11eaf67a00d0669 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 11:16:11 -0600 Subject: [PATCH 113/125] Disable rules for build container job for now --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e666a6149..9ea2ca4a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,9 +50,10 @@ variables: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" - changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + # disabled for troubleshooting + # rules: + # - if: $CI_PIPELINE_SOURCE == "push" + # changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] # disabled for troubleshooting From 77809791213182bd36767fed94e4e1611e0f1586 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 12:43:44 -0600 Subject: [PATCH 114/125] Organize job into stages This should make the pipeline summary icons in the GitLab UI more useful. --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ea2ca4a6..0717d1b2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,8 @@ default: when: always untracked: true +stages: [ build-container, build-gridpack, test-gridpack ] + variables: KUBERNETES_CPU_REQUEST: "1" @@ -16,6 +18,7 @@ variables: # https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy .build-container: + stage: build-container timeout: 5 hours image: name: gcr.io/kaniko-project/executor:v1.9.0-debug @@ -72,6 +75,7 @@ build-container:rockylinux: .build-gridpack: + stage: build-gridpack variables: MAKE_JOBS: "2" script: @@ -96,6 +100,7 @@ build-gridpack:rockylinux: .test-gridpack: + stage: test-gridpack script: - useradd gridpack - chown -R gridpack:gridpack . From 240306afeeb54cd62fe959063359e3c48332206b Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 13:50:43 -0600 Subject: [PATCH 115/125] Add epel repo and enable crb repo For python3-mpi4py-openmpi and its deps --- install_environment_packages.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 162771989..6cc945f05 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -19,8 +19,11 @@ debian | ubuntu) ;; fedora | rhel | centos | rocky) dnf upgrade --assumeyes --verbose && + dnf install epel-release --assumeyes && + crb enable && dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf python3-mpi4py-openmpi + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi ;; *) echo "$distribution not supported" From 11df6b1d5f7508fa36f968a39663107de1de6df7 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 13:54:29 -0600 Subject: [PATCH 116/125] Remove troubleshooting commands --- .gitlab-ci.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0717d1b2b..f3913b51c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,25 +26,7 @@ variables: variables: IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env script: - - mkdir -p /kaniko/.docker - - | - cat << EOF > /kaniko/.docker/config.json - { - "proxies": { - "default": { - "httpProxy": "${http_proxy}", - "httpsProxy": "${https_proxy}" - } - } - } - EOF - - if [ -f /etc/gitlab-runner/certs/ca.crt ]; then - cat /etc/gitlab-runner/certs/ca.crt >> /kaniko/ssl/certs/ca-certificates.crt - fi - - chmod +x *.sh - - env - > /kaniko/executor --context "${CI_PROJECT_DIR}" From 20051ddd4504aa2e976ea1793fc08cb4f9cec2ea Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 17:50:34 -0600 Subject: [PATCH 117/125] Normalize ENV declarations --- dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dockerfile b/dockerfile index 92bb67862..5ec6dc9f5 100644 --- a/dockerfile +++ b/dockerfile @@ -2,9 +2,9 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ENV GP_EXT_DEPS=/gridpack-dependencies -ENV BOOST_VERSION "1.78.0" -ENV GA_VERSION "5.8" -ENV PETSC_VERSION "3.16.4" +ENV BOOST_VERSION="1.78.0" +ENV GA_VERSION="5.8" +ENV PETSC_VERSION="3.16.4" ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} ARG http_proxy From c580ba2971134aad5c78570ea818052d0e31b49d Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:15:02 -0600 Subject: [PATCH 118/125] Formatting changes --- install_environment_packages.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index 6cc945f05..e763a31a6 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -12,18 +12,29 @@ distribution=$( case $distribution in debian | ubuntu) - apt update && - apt upgrade -y && - apt install -y \ - wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py + + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py ;; + fedora | rhel | centos | rocky) - dnf upgrade --assumeyes --verbose && - dnf install epel-release --assumeyes && - crb enable && - dnf install --assumeyes \ - wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi + + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi + ;; *) echo "$distribution not supported" From 08e019a87b5ab2baf59d935efdc5badaa63c9938 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:15:31 -0600 Subject: [PATCH 119/125] Add mpicc to path after openmpi-devel install --- install_environment_packages.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index e763a31a6..ea53fcd9e 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -35,6 +35,9 @@ fedora | rhel | centos | rocky) wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ python3-mpi4py-openmpi + # add mpicc to path (from openmpi-devel) + export PATH=$PATH:/usr/lib64/openmpi/bin + ;; *) echo "$distribution not supported" From a3773421545906998951db66c7b505f8612ea484 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 19:16:00 -0600 Subject: [PATCH 120/125] Reenable ubuntu build --- .gitlab-ci.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3913b51c..d09c8a2e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ default: when: always untracked: true + stages: [ build-container, build-gridpack, test-gridpack ] @@ -35,14 +36,12 @@ variables: --build-arg "https_proxy=$https_proxy" --dockerfile "${CI_PROJECT_DIR}/dockerfile" --destination "${IMAGE_PATH}" > build.log 2>&1 - # disabled for troubleshooting - # rules: - # - if: $CI_PIPELINE_SOURCE == "push" - # changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] -# disabled for troubleshooting -.build-container:ubuntu: +build-container:ubuntu: extends: .build-container variables: BASE_IMAGE: ubuntu @@ -64,8 +63,7 @@ build-container:rockylinux: - ./install_gridpack.sh -# disabled for troubleshooting -.build-gridpack:ubuntu: +build-gridpack:ubuntu: extends: .build-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: @@ -91,8 +89,7 @@ build-gridpack:rockylinux: paths: [ src/build/Testing/Temporary ] -# disabled for troubleshooting -. test-gridpack:ubuntu: +test-gridpack:ubuntu: extends: .test-gridpack image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env needs: [ build-gridpack:ubuntu ] From b66c87ae56fa42500ae808bb7e1f9647b8334af2 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 21:50:30 -0600 Subject: [PATCH 121/125] Add mpicc to path if on RHEL --- install_gridpack_deps.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index f0e0fa310..37d63427a 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -75,6 +75,19 @@ function install_ga { pushd ga || exit + # check if we are running on RHEL + distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" + ) + case $distribution in + fedora | rhel | centos | rocky) + # add mpicc to path (from openmpi-devel) + export PATH=$PATH:/usr/lib64/openmpi/bin + ;; + esac + # build echo "Configuring Global Arrays" ./configure \ From c2089f13939317a68c3a185afebaf6f9c7ff67ae Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Tue, 23 Jan 2024 21:52:35 -0600 Subject: [PATCH 122/125] Remove path mod for mpicc --- install_environment_packages.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index ea53fcd9e..eb5fa910c 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -34,10 +34,6 @@ fedora | rhel | centos | rocky) dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ python3-mpi4py-openmpi - - # add mpicc to path (from openmpi-devel) - export PATH=$PATH:/usr/lib64/openmpi/bin - ;; *) echo "$distribution not supported" From bdebd000f57e4c723239e328792903f00efeee23 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 09:51:32 -0600 Subject: [PATCH 123/125] Load mpi module for RHEL --- install_gridpack.sh | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/install_gridpack.sh b/install_gridpack.sh index 0b0e17140..f95c8fe08 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -13,7 +13,8 @@ function install_gridpack { local gridpack_deps_dir=$1 local gridpack_build_dir=$2 local gridpack_install_dir=$3 - : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local distribution=$4 + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" # remove existing build and install dir rm -rf "$gridpack_build_dir" @@ -29,6 +30,15 @@ function install_gridpack { # remove existing cmake output rm -rf CMake* + + # load module related to mpi4py + case $distribution in + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; + esac + # generate make files echo "Generating GridPACK make files" cmake \ @@ -65,7 +75,8 @@ function install_gridpack_python { # args local gridpack_build_dir=$1 local gridpack_install_dir=$2 - : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" + local distribution=$3 + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" # update submodules echo "Updating GridPACK submodules" @@ -74,11 +85,6 @@ function install_gridpack_python { pushd python || exit # set an env var if we are running on RHEL - distribution=$( - # shellcheck source=/dev/null - source /etc/os-release - echo "$ID" - ) case $distribution in fedora | rhel | centos | rocky) export RHEL_OPENMPI_HACK=yes @@ -121,7 +127,13 @@ date build_dir=${PWD}/src/build install_dir=${PWD}/src/install -install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" -install_gridpack_python "$build_dir" "$install_dir" +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" "$distribution" +install_gridpack_python "$build_dir" "$install_dir" "$distribution" echo "Completed GridPACK installation" From 96214c0945007647474690a33d531ae2be847da5 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 09:51:52 -0600 Subject: [PATCH 124/125] Load mpi module instead of altering PATH --- install_gridpack_deps.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 37d63427a..75740acd8 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -82,10 +82,10 @@ function install_ga { echo "$ID" ) case $distribution in - fedora | rhel | centos | rocky) - # add mpicc to path (from openmpi-devel) - export PATH=$PATH:/usr/lib64/openmpi/bin - ;; + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; esac # build From d85e3f156897eab58fa33b5a764cefdb44739535 Mon Sep 17 00:00:00 2001 From: Jack Savage Date: Wed, 24 Jan 2024 10:52:24 -0600 Subject: [PATCH 125/125] Add boost-openmpi package to rocky linux --- install_environment_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_environment_packages.sh b/install_environment_packages.sh index eb5fa910c..211d962c4 100644 --- a/install_environment_packages.sh +++ b/install_environment_packages.sh @@ -33,7 +33,7 @@ fedora | rhel | centos | rocky) # install required packages dnf install --assumeyes \ wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ - python3-mpi4py-openmpi + python3-mpi4py-openmpi boost-openmpi ;; *) echo "$distribution not supported"