diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..d09c8a2e6 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,101 @@ +default: + tags: [ basic, gridpack, ikp, k8s ] + artifacts: + when: always + untracked: true + + +stages: [ build-container, build-gridpack, test-gridpack ] + + +variables: + KUBERNETES_CPU_REQUEST: "1" + KUBERNETES_CPU_LIMIT: "2" + KUBERNETES_MEMORY_REQUEST: "4Gi" + KUBERNETES_MEMORY_LIMIT: "8Gi" + http_proxy: "http://proxy01.pnl.gov:3128" + https_proxy: ${http_proxy} + + +# https://docs.gitlab.com/ee/ci/docker/using_kaniko.html#building-an-image-with-kaniko-behind-a-proxy +.build-container: + stage: build-container + timeout: 5 hours + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + variables: + IMAGE_PATH: ${CI_REGISTRY_IMAGE}:${BASE_IMAGE}-gridpack-env + script: + - chmod +x *.sh + - > + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --build-arg "BASE_IMAGE=${BASE_IMAGE}:${BASE_IMAGE_TAG}" + --build-arg "http_proxy=$http_proxy" + --build-arg "https_proxy=$https_proxy" + --dockerfile "${CI_PROJECT_DIR}/dockerfile" + --destination "${IMAGE_PATH}" > build.log 2>&1 + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: [ "dockerfile", "install_environment_packages.sh", "install_gridpack_deps.sh" ] + + +build-container:ubuntu: + extends: .build-container + variables: + BASE_IMAGE: ubuntu + BASE_IMAGE_TAG: "22.04" + + +build-container:rockylinux: + extends: .build-container + variables: + BASE_IMAGE: rockylinux + BASE_IMAGE_TAG: "9" + + +.build-gridpack: + stage: build-gridpack + variables: + MAKE_JOBS: "2" + script: + - ./install_gridpack.sh + + +build-gridpack:ubuntu: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: + - job: build-container:ubuntu + optional: true + + +build-gridpack:rockylinux: + extends: .build-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: + - job: build-container:rockylinux + optional: true + + +.test-gridpack: + stage: test-gridpack + script: + - useradd gridpack + - chown -R gridpack:gridpack . + - su gridpack -c 'ctest --test-dir src/build --output-on-failure' + artifacts: + paths: [ src/build/Testing/Temporary ] + + +test-gridpack:ubuntu: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:ubuntu-gridpack-env + needs: [ build-gridpack:ubuntu ] + + +test-gridpack:rockylinux: + extends: .test-gridpack + image: ${CI_REGISTRY_IMAGE}:rockylinux-gridpack-env + needs: [ build-gridpack:rockylinux ] diff --git a/README.md b/README.md index 08d243154..5b7422099 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ # GridPACKTM--> # GridPACK: High-Performance Electric Grid Simulation +![PNNL Build Status](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.github.com%2Frepos%2FGridOPTICS%2FGridPACK%2Fcommits%2Fdevelop%2Fstatus&query=%24.state&logo=gitlab&label=build%20status&color=white +) + GridPACK is an open-source high-performance (HPC) package for simulation of large-scale electrical grids. Powered by distributed (parallel) computing and high-performance numerical solvers, GridPACK offers several applications forfast simulation of electrical transmission systems. GridPACK includes a number of prebuilt applications that can be directly used. The most commonly used and well-developed are: - AC Power Flow - Dynamics Simulation @@ -18,7 +21,7 @@ In addition, GridPACK is also a framework to simplify the development of new app See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms. ## Usage -See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. +See [User manual](docs/user_manual/GridPACK.pdf) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. - Quick Guide (To do) @@ -36,16 +39,16 @@ The best (and fastest) way to reach us for any technical questions is by posting ## Citing GridPACK ``` -@article{doi:10.1177/1094342015607609, -author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, -title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, -journal = {The International Journal of High Performance Computing Applications}, -volume = {30}, -number = {2}, -pages = {223-240}, -year = {2016}, -doi = {10.1177/1094342015607609}, -URL = {https://doi.org/10.1177/1094342015607609}, +@article{doi:10.1177/1094342015607609, +author = {Bruce Palmer and William Perkins and Yousu Chen and Shuangshuang Jin and David C allahan and Kevin Glass and Ruisheng Diao and Mark Rice and Stephen Elbert and Mallikarjun a Vallem and Zhenyu Huang}, +title ={GridPACKTM: A framework for developing power grid simulations on high-performance computing platforms}, +journal = {The International Journal of High Performance Computing Applications}, +volume = {30}, +number = {2}, +pages = {223-240}, +year = {2016}, +doi = {10.1177/1094342015607609}, +URL = {https://doi.org/10.1177/1094342015607609}, eprint = {https://doi.org/10.1177/1094342015607609} ``` @@ -69,7 +72,7 @@ GridPACK has been developed through funding from various sources over the years. ## Copyright Copyright © 2013, Battelle Memorial Institute. -GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. +GridPACKTM is a free software distributed under a BSD 2-clause license. You may reuse, modify, and redistribute the software. See the [license](src/LICENSE.md) file for details. diff --git a/dockerfile b/dockerfile new file mode 100644 index 000000000..5ec6dc9f5 --- /dev/null +++ b/dockerfile @@ -0,0 +1,21 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +ENV GP_EXT_DEPS=/gridpack-dependencies +ENV BOOST_VERSION="1.78.0" +ENV GA_VERSION="5.8" +ENV PETSC_VERSION="3.16.4" +ENV LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost/install_for_gridpack/lib:${GP_EXT_DEPS}/ga/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} + +ARG http_proxy +ARG https_proxy +ENV http_proxy=${http_proxy} +ENV https_proxy=${https_proxy} + +WORKDIR ${GP_EXT_DEPS} + +COPY install_environment_packages.sh . +RUN ./install_environment_packages.sh && rm *.sh + +COPY install_gridpack_deps.sh . +RUN ./install_gridpack_deps.sh && rm *.sh diff --git a/install_environment_packages.sh b/install_environment_packages.sh new file mode 100644 index 000000000..211d962c4 --- /dev/null +++ b/install_environment_packages.sh @@ -0,0 +1,42 @@ +#! /bin/bash + +# installs necessary packages for a given distribution + +set -xeuo pipefail + +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +case $distribution in +debian | ubuntu) + + apt-get update + apt-get upgrade --yes + + # install required packages + apt-get install --yes \ + wget build-essential git python3.11 python3-pip libopenmpi-dev cmake pkg-config python3-mpi4py + ;; + +fedora | rhel | centos | rocky) + + dnf upgrade --assumeyes --verbose + + # use epel and crb repos + # https://wiki.rockylinux.org/rocky/repo/#notes-on-epel + dnf install epel-release --assumeyes + crb enable + + # install required packages + dnf install --assumeyes \ + wget @development git python3.11 python3-pip openmpi-devel cmake pkgconf \ + python3-mpi4py-openmpi boost-openmpi + ;; +*) + echo "$distribution not supported" + exit 1 + ;; +esac diff --git a/install_gridpack.sh b/install_gridpack.sh index 933636dae..f95c8fe08 100755 --- a/install_gridpack.sh +++ b/install_gridpack.sh @@ -1,103 +1,139 @@ -# This script installs GridPACK and python wrappter.GridPACK is built in src/build directory and installed in src/install directory. +#! /bin/bash -# This script should be run from the top-level GridPACK directory. +# installs GridPACK and its python wrapper +# gridPACK is built in src/build and installed to src/install +# run this from the top level GridPACK directory -# Flag for install GridPACK and GridPACK python wrapper -install_gridpack=true -install_gridpack_python=true +set -xeuo pipefail -# Set your python executable here -python_exe=`which python` -if test -z ${python_exe} -then - python_exe=`which python3` -fi +function install_gridpack { + echo "--- GridPACK ---" -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi + # args + local gridpack_deps_dir=$1 + local gridpack_build_dir=$2 + local gridpack_install_dir=$3 + local distribution=$4 + : "${gridpack_deps_dir:?}" "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" -# Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies -fi + # remove existing build and install dir + rm -rf "$gridpack_build_dir" + rm -rf "$gridpack_install_dir" -cd ${GRIDPACK_ROOT_DIR} + # create the build dir + mkdir -p "$gridpack_build_dir" -# Set environment variable GRIDPACK_BUILD_DIR and create a build directory -export GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build + pushd "$gridpack_build_dir" || exit -GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install -export GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} - -if ${install_gridpack} -then - - rm -rf $GRIDPACK_BUILD_DIR - mkdir $GRIDPACK_BUILD_DIR - - rm -rf ${GRIDPACK_INSTALL_DIR} - - cd ${GRIDPACK_BUILD_DIR} - - ## GridPACK installation - echo "Building GridPACK" - -# git checkout develop + # todo? git checkout develop + # remove existing cmake output rm -rf CMake* - cmake_args="-D GA_DIR:STRING=${GP_EXT_DEPS}/ga-5.8/install_for_gridpack \ - -D BOOST_ROOT:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack \ - -D Boost_DIR:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib/cmake/Boost-1.78.0 \ - -D Boost_LIBRARIES:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib \ - -D Boost_INCLUDE_DIRS:STRING=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/include \ - -D PETSC_DIR:PATH=${GP_EXT_DEPS}/petsc/install_for_gridpack \ - -D MPI_CXX_COMPILER:STRING='mpicxx' \ - -D MPI_C_COMPILER:STRING='mpicc' \ - -D MPIEXEC:STRING='mpiexec' \ - -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ - -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ - -D CMAKE_BUILD_TYPE:STRING=Debug \ - -D BUILD_SHARED_LIBS=YES \ - -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ - .. " - - cmake ${cmake_args} - - echo "Installing GridPACK develop branch" - make -j 10 install -fi - -if ${install_gridpack_python} -then - echo "Installing GridPACK python wrapper" - - cd ${GRIDPACK_ROOT_DIR} - - git submodule update --init - - echo ${GRIDPACK_DIR} - cd python - - export RHEL_OPENMPI_HACK=yes - - rm -rf build - - ${python_exe} setup.py build - - rm -rf ${GRIDPACK_INSTALL_DIR}/lib/python - mkdir ${GRIDPACK_INSTALL_DIR}/lib/python - - PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}" - export PYTHONPATH - ${python_exe} setup.py install --home="$GRIDPACK_DIR" - -fi - -cd ${GRIDPACK_ROOT_DIR} + + # load module related to mpi4py + case $distribution in + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; + esac + + # generate make files + echo "Generating GridPACK make files" + cmake \ + -D GA_DIR:STRING="${gridpack_deps_dir}/ga/install_for_gridpack" \ + -D BOOST_ROOT:STRING="${gridpack_deps_dir}/boost/install_for_gridpack" \ + -D Boost_DIR:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib/cmake/Boost" \ + -D Boost_LIBRARIES:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/lib" \ + -D Boost_INCLUDE_DIRS:STRING="${gridpack_deps_dir}/boost/install_for_gridpack/include" \ + -D PETSC_DIR:PATH="${gridpack_deps_dir}/petsc/install_for_gridpack" \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D GRIDPACK_TEST_TIMEOUT:STRING=30 \ + -D CMAKE_INSTALL_PREFIX:PATH="${gridpack_install_dir}" \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=YES \ + -D Boost_NO_SYSTEM_PATHS:BOOL=TRUE \ + -D Boost_NO_BOOST_CMAKE:BOOL=TRUE \ + -D MPIEXEC_MAX_NUMPROCS:STRING="2" \ + .. + + # install + echo "Installing GridPACK" + make -j "${MAKE_JOBS:-$(nproc)}" install + + popd || exit + + echo "GridPACK installation complete" +} + +function install_gridpack_python { + echo "--- GridPACK python wrapper ---" + + # args + local gridpack_build_dir=$1 + local gridpack_install_dir=$2 + local distribution=$3 + : "${gridpack_build_dir:?}" "${gridpack_install_dir:?}" "${distribution:?}" + + # update submodules + echo "Updating GridPACK submodules" + git submodule update --init + + pushd python || exit + + # set an env var if we are running on RHEL + case $distribution in + fedora | rhel | centos | rocky) + export RHEL_OPENMPI_HACK=yes + ;; + esac + + # set python executable path + python_exe=$(which python || which python3) + + # remove existing build dir + rm -rf build + + # export GRIDPACK_DIR + export GRIDPACK_DIR="${gridpack_install_dir}" + + # build + echo "Building GridPACK python wrapper" + ${python_exe} setup.py build + + # remove existing install dir + local py_lib="${gridpack_install_dir}/lib/python" + rm -rf "${py_lib}" + mkdir -p "${py_lib}" + + # add lib to python path + export PYTHONPATH="${py_lib}${PYTHONPATH:+:$PYTHONPATH}" + + # install + echo "Installing GridPACK python wrapper" + ${python_exe} setup.py install --home="$gridpack_install_dir" + + popd || exit + + echo "GridPACK python wrapper installation complete" +} + +echo "Installing GridPACK" +date + +build_dir=${PWD}/src/build +install_dir=${PWD}/src/install + +distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" +) + +install_gridpack "${GP_EXT_DEPS:?}" "$build_dir" "$install_dir" "$distribution" +install_gridpack_python "$build_dir" "$install_dir" "$distribution" echo "Completed GridPACK installation" diff --git a/install_gridpack_deps.sh b/install_gridpack_deps.sh index 87897593e..75740acd8 100755 --- a/install_gridpack_deps.sh +++ b/install_gridpack_deps.sh @@ -1,133 +1,168 @@ -# This script installs all GridPACK dependencies.The dependencies are installed in external-dependencies directory. - -# This script should be run from the top-level GridPACK directory. - -# Flags for installing/not installing different dependency packages -install_boost=true -install_ga=true -install_petsc=true - -echo $(date) - -if test -z ${GRIDPACK_ROOT_DIR} -then - export GRIDPACK_ROOT_DIR=${PWD} - echo "GRIDPACK_ROOT_DIR = ${GRIDPACK_ROOT_DIR}" -fi - -cd ${GRIDPACK_ROOT_DIR} - -# Create directory for installing external packages -if test -z ${GP_EXT_DEPS} -then - export GP_EXT_DEPS=${GRIDPACK_ROOT_DIR}/external-dependencies - rm -rf ${GP_EXT_DEPS} - mkdir ${GP_EXT_DEPS} - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" -else - if test -d ${GP_EXT_DEPS} - then - echo "GRIDPACK_EXT_DEPENDENCIES_DIR=${GP_EXT_DEPS}" - else - mkdir ${GP_EXT_DEPS} - fi -fi - -cd ${GP_EXT_DEPS} - -if ${install_boost} -then +#! /bin/bash - rm -rf boost* - - # Download and install Boost - echo "Downloading Boost-1.78.0" +# installs GridPACK dependencies to the current directory + +set -xeuo pipefail + +function install_boost { + local boost_version=$1 - # Download Boost - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz + # check args + : "${boost_version:?}" - # Untar - tar -xf boost_1_78_0.tar.gz + echo "--- Installing Boost ${boost_version} ---" + + # remove existing + rm -rf boost* - cd boost_1_78_0 + # download + echo "Downloading Boost" + wget \ + "https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version//./_}.tar.gz" \ + -O boost.tar.gz \ + --quiet - # Install boost - echo "Building Boost-1.78.0" + # unpack + echo "Unpacking Boost" + tar -xf boost.tar.gz && rm boost.tar.gz - ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system + # remove version from dir name + mv "boost_${boost_version//./_}" boost - echo 'using mpi ;' >> project-config.jam + pushd boost || exit + # bootstrap + echo "Bootstrapping Boost" + ./bootstrap.sh \ + --prefix=install_for_gridpack \ + --with-libraries=mpi,serialization,random,filesystem,system + echo 'using mpi ;' >>project-config.jam + + # build + echo "Building Boost" ./b2 -a -d+2 link=shared stage - echo "Installing Boost-1.78.0" + # install + echo "Installing Boost" ./b2 -a -d+2 link=shared install - echo "Building and Installing Boost libraries complete" -fi + popd || exit + + echo "Boost installation complete" +} + +function install_ga { + local ga_version=$1 + + # check args + : "${ga_version:?}" -if ${install_ga} -then - # Download, build, and install GA - cd ${GP_EXT_DEPS} + echo "--- Installing Global Arrays ${ga_version} ---" - echo "Downloading GA-5.8" + # download + echo "Downloading Global Arrays" + wget \ + "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" \ + -O ga.tar.gz \ + --quiet - wget https://github.com/GlobalArrays/ga/releases/download/v5.8/ga-5.8.tar.gz + # unpack + echo "Unpacking Global Arrays" + tar -xf ga.tar.gz && rm ga.tar.gz - tar -xf ga-5.8.tar.gz + # remove version from dir name + mv "ga-${ga_version}" ga - cd ga-5.8 + pushd ga || exit - # Build GA - echo "Building GA-5.8" - ./configure --with-mpi-ts --disable-f77 --without-blas --enable-cxx --enable-i4 --prefix=${PWD}/install_for_gridpack --enable-shared - - # Install GA - echo "Installing GA-5.8" - make -j 10 install - - echo "GA-5.8 installation complete" -fi + # check if we are running on RHEL + distribution=$( + # shellcheck source=/dev/null + source /etc/os-release + echo "$ID" + ) + case $distribution in + fedora | rhel | centos | rocky) + echo "Loading mpi4py module" + module load "mpi/openmpi-$(arch)" + ;; + esac -if ${install_petsc} -then - - # Install PETSc 3.16.4 - cd ${GP_EXT_DEPS} + # build + echo "Configuring Global Arrays" + ./configure \ + --with-mpi-ts \ + --disable-f77 \ + --without-blas \ + --enable-cxx \ + --enable-i4 \ + --prefix="${PWD}/install_for_gridpack" \ + --enable-shared - # Download - echo "Downloading PETSc 3.16.4" + # install + echo "Installing Global Arrays" + make -j "$(nproc)" install + popd || exit + + echo "Global Arrays installation complete" +} + +function install_petsc { + local petsc_version=$1 + + # check args + : "${petsc_version:?}" + + echo "--- Installing PETSc ${petsc_version} ---" + + # clone + echo "Cloning PETSc repository" git clone https://gitlab.com/petsc/petsc.git - - cd petsc - git checkout tags/v3.16.4 -b v3.16.4 + pushd petsc || exit + + git checkout "tags/v${petsc_version}" -b "v${petsc_version}" export PETSC_DIR=${PWD} export PETSC_ARCH=build-dir - # Install PETSc - echo "Installing PETSc 3.16.4" - - ./configure --download-superlu_dist --download-metis --download-parmetis --download-suitesparse --download-f2cblaslapack --download-cmake --prefix=${PWD}/install_for_gridpack --scalar-type=complex --with-shared-libraries=1 --download-f2cblaslapack=1 - - # Build PETSc - echo "Building PETSc 3.16.4" + # install + echo "Configuring PETSc" + ./configure \ + --download-mumps \ + --download-scalapack \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --download-cmake \ + --prefix="${PWD}"/install_for_gridpack \ + --scalar-type=complex \ + --with-shared-libraries=1 + + # build + echo "Building PETSc" + make + + # install + echo "Installing PETSc" + make install - make + # check + echo "Checking PETSc" + make check - # Install PETSc - echo "Installing PETSc 3.16.4" + popd || exit - make install - make check -fi + echo "PETSc installation complete" +} -# update LD_LIBRARY_PATH so that boost,ga, and petsc are on it -export LD_LIBRARY_PATH=${GP_EXT_DEPS}/boost_1_78_0/install_for_gridpack/lib:${GP_EXT_DEPS}/ga-5.8/install_for_gridpack/lib:${GP_EXT_DEPS}/petsc/install_for_gridpack/lib:${LD_LIBRARY_PATH} +echo "Installing GridPACK dependencies" +date -cd ${GRIDPACK_ROOT_DIR} +install_boost "${BOOST_VERSION:?}" +install_ga "${GA_VERSION:?}" +install_petsc "${PETSC_VERSION:?}" -echo "Completed installing GridPACK dependencies in ${GP_EXT_DEPS}" +echo "GridPACK dependency installation complete"