Merge MPI version with changes in develop

.github/workflows/ci.yml

@@ -7,9 +7,10 @@ on:
 
 jobs:
   test:
-    name: TROVE ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.compiler }}
+    name: TROVE ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.compiler }} - MPI:${{ matrix.mpi }}
     env:
       OMP_NUM_THREADS: 1
+      USE_MPI: ${{ matrix.mpi && 1 || 0 }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -20,16 +21,40 @@ jobs:
           - x64
         compiler:
           - gfortran
+          - intel
+        mpi:
+          - true
+          - false
+        exclude:
+          # only use MPI with intel
+          - compiler: gfortran
+            mpi: true
     steps:
       - uses: actions/checkout@v2
 
-      #- name: Cache benchmark data
-        #uses: actions/cache@v2
-        #env:
-          #cache-name: benchmark-data
-        #with:
-          #path: test/benchmarks
-          #key: benchmark-data
+      - name: Restored cached Intel install
+        id: cache_intel
+        if: ${{ matrix.compiler == 'intel' }}
+        uses: actions/cache@v2
+        with:
+          key: oneapi-${{ matrix.os }}-${{ matrix.arch }}-MPI:${{ matrix.mpi }}
+          path: |
+            /opt/intel/oneapi/*
+            !/opt/intel/oneapi/conda_channel
+            !/opt/intel/oneapi/debugger
+            !/opt/intel/oneapi/licensing
+
+      - name: Setup Intel compiler and MPI
+        if: ${{ matrix.compiler  == 'intel' && steps.cache_intel.outputs.cache-hit != 'true'}}
+        run: |
+          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+          echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+          sudo apt-get update -o Dir::Etc::sourcelist="sources.list.d/oneAPI.list" -o APT::Get::List-Cleanup="0"
+          sudo apt-get install -y intel-oneapi-compiler-fortran
+          if [ ${USE_MPI} = 1 ]; then
+            sudo apt-get install -y intel-oneapi-mpi-devel intel-oneapi-mkl-devel
+          fi
 
       - name: Setup python
         uses: actions/setup-python@v1
@@ -38,15 +63,39 @@ jobs:
           architecture: x64
 
       - name: Install Intel MKL
+        # MKL will have been installed alongside MPI if we're using that
+        if: ${{ !matrix.mpi }}
         run: sudo apt-get install -y intel-mkl
 
-      - name: Install pipenv environment
-        run: cd test && pip install pipenv && pipenv install
-
-      - name: Build
+      - name: Build (gfortran)
+        if: ${{ matrix.compiler == 'gfortran' }}
         run: |
           gfortran --version
           make COMPILER=gfortran MODE=ci
 
+      - name: Build (intel)
+        if: ${{ matrix.compiler == 'intel' }}
+        run: |
+          source /opt/intel/oneapi/compiler/latest/env/vars.sh
+          source /opt/intel/oneapi/setvars.sh
+          if [ -d "/opt/intel/oneapi/mkl" ]; then
+            source /opt/intel/oneapi/mkl/latest/env/vars.sh
+          fi
+          # mpif90 uses gfortran by default unless changed like this
+          export I_MPI_F90=ifort
+          ifort --version
+          make COMPILER=intel MODE=ci USE_MPI=${USE_MPI}
+
       - name: Test
-        run: make test
+        env:
+          # nproc is used in the test script for the number of processes
+          nproc: ${{ matrix.mpi && 2 || 1 }}
+        run: |
+          if [[ -d "/opt/intel/oneapi" ]]; then
+            source /opt/intel/oneapi/compiler/latest/env/vars.sh
+            source /opt/intel/oneapi/setvars.sh
+            if [ ${USE_MPI} = 1 ]; then
+               source /opt/intel/oneapi/mkl/latest/env/vars.sh
+            fi
+          fi
+          make test

TROVE.wiki/TROVE-mpi.md

@@ -0,0 +1,82 @@
+# MPI functionality
+
+Distributed memory parallelism using MPI has been implemented in the
+PTcontracted_matalem_class subroutine, plus subroutines called by or related to
+it. Most MPI functionality has been consolidated in the mpi_aux.f90 file.
+
+File output has been implemented using MPI-IO. Because MPI is not compatible
+with the Fortran's record-based unformatted file format, a new file format has
+been implemented. Preserving the old file format by using Master I/O has been
+considered, but cannot be supported for large simulations (where distributed
+memory parallelism is required) as the size of the data needing to be written
+will exceed the available memory on a compute node.
+
+The new file format is similar to the non-mpi format, except for using 'raw'
+binary data versus Fortran's records. Conversion utilities between the two
+formats will be provided. 
+
+The following has been implemented:
+
+ - Writing of single-file data
+ - Reading of single-file data
+ - Writing of split file data
+
+The following needs testing:
+
+ - Reading of split file data
+
+The following is in progress:
+
+ - Reading the new format on MPI-free systems
+ - Conversion functionality
+
+
+To enable writing in the new file format, add the parameter `format		mpiio` to
+the input file under `CHECK_POINT`, e.g.:
+
+```
+CHECK_POINT
+HAMILTONIAN read
+potential   read
+kinetic     read
+external    read
+basis_set   read
+CONTRACT    read
+matelem     save
+extmatelem  save
+eigenfunc   save
+format			mpiio
+END
+```
+
+NOTE: As other routines than `matelem` have not been touched yet, expect a large
+amount of duplicated output from stdout. This does not affect the integrity of
+file output.
+
+# How to run
+
+## Requirements
+
+The only additional requirement to build and run the MPI version of TROVE is a
+compatible MPI library. The code has been tested with Intel MPI, versions
+2017.4, 2018.4 and 2019.3. Intel 2018.1 is confirmed *not* working.
+
+This version of TROVE should also build and run correctly with OpenMPI, and
+should be compatible with GNU GFortran but this is untested.
+
+Although not required, it is highly recommended to run on a distributed
+filesystem (e.g. Lustre + striping) for significantly increased I/O performance.
+
+## Building TROVE-MPI
+
+Make sure the compiler is set to 'mpif90' or 'mpifort' in the Makegfile, then
+run `make` in the source directory.
+
+## Running TROVE-MPI
+
+Once built, run the code like the sequential version but prepended with `mpirun`
+or your scheduler's mpi wrapper (e.g. `srun`). E.g.:
+
+`mpirun trove.x infile.inp`
+
+A sample batch script for SLURM is provided.

TROVE.wiki/trove-slurm.batch

@@ -0,0 +1,30 @@
+#!/bin/bash
+#SBATCH -J TROVE-MPI-<input>
+#SBATCH -p <PARTIITON>
+#SBATCH -A <ACCOUNT>
+#SBATCH -N <NUM_NODES>
+#SBATCH -n <NUM_TASKS>      # Recommend number of sockets
+#SBATCH -c <CORES_PER_TASK> # for OpenMP, NOTE recommend number of cores per socket
+#SBATCH -t 3:0:0
+#SBATCH --exclusive
+#SBATCH -o "trove.%j.out"
+
+# This batch script runs TROVE-MPI using hybrid MPI/OpenMP. For best
+# performance, it is recommended to run one MPI task per socket, and <ncores per
+# socket> OpenMP threads per task.
+
+module purge
+module load rhel7/default-peta4
+module unload intel/bundles/complib/2017.4
+module load intel/bundles/complib/2019.3
+
+# Set to number of cores per socket
+export OMP_NUM_THREADS=16
+
+# Prevent processes from migrating and ensure each task + threads is exclusive
+# to one socket.
+export I_MPI_PIN=1
+export I_MPI_PIN_DOMAIN=omp:compact
+
+
+mpirun trove.x file1.inp

accuracy.f90

@@ -15,8 +15,8 @@ module accuracy
   integer, parameter :: drk         = selected_real_kind(12,25)  ! "Double" reals and complex (complexi? :-)
   integer, parameter :: rk          = selected_real_kind(12,25)  ! "Normal" reals and complex (complexi? :-)
   integer, parameter :: ark         = selected_real_kind(25,32)  ! "Accurate" reals and complex (complexi? :-)
-  integer, parameter :: inp         = 5                          ! Output I/O channel
-  integer, parameter :: out         = 6                          ! Output I/O channel
+  integer, parameter :: inp         = 5                          ! Input I/O channel
+  integer            :: out         = 6                          ! Output I/O channel
   integer, parameter :: nfilelegendre = 101                      ! Dump-outout channel for eigenfunction 
 
   ! universal constants

diag.f90

@@ -6,7 +6,6 @@ module diag
 #define omparpack_  0
 #define propack_  0
 !
-!
 !  Simplistic type-agnostic LAPACK interface
 !
   use accuracy
@@ -15,7 +14,7 @@ module diag
   private verbose
   !
   !
-  public diag_tridiag,diag_tridiag_pack,diag_dsyev_i8,diag_dgelss,diag_syev_ilp,diag_syev_i8,diag_dseupd,diag_dseupd_p
+  public diag_tridiag,diag_tridiag_pack,diag_dsyev_i8,diag_dgelss,diag_syev_ilp,diag_syev_i8
   public dseupd_omp_arpack,diag_propack,daprod
   !
   integer,parameter:: verbose = 4
@@ -2094,7 +2093,7 @@ subroutine diag_dseupd(n,bterm,nroots,factor,maxitr_,tol,h,e)
            call dsaupd ( ido, bmat, n, which, nev, tol, resid, &
                          ncv, v, ldv, iparam, ipntr, workd, workl, &
                          lworkl, info )
-#else 
+#else
             !
             write(out,"(/'Arpack was not activated yet. Please uncomment dsaupd and  dseupd bellow')")
             stop 'Arpack was not activated'
@@ -3274,9 +3273,6 @@ subroutine dseupd_omp_arpack(n,bterm,nroots,factor,maxitr_,tol,h,e)
   end subroutine dseupd_omp_arpack
 
 
-
-
-
   subroutine diag_propack(n,bterm,nroots,factor,maxiter,iverbose,tol,h,e)
     !
     integer,intent(in)     :: n,bterm(n,2),maxiter,iverbose