Parallel I/O integration

CMakeLists.txt

@@ -7,14 +7,43 @@ project(
 )
 
 option(CABLE_MPI "Build the MPI executable" OFF)
+option(CABLE_TESTS "Build CABLE tests" OFF)
 
 # third party libs
 if(CABLE_MPI)
     find_package(MPI REQUIRED COMPONENTS Fortran)
+    find_package(PIO COMPONENTS Fortran QUIET)
+    if(TARGET PIO::PIO_Fortran)
+        message(STATUS "Found PIO_Fortran: ${PIO_DIR}")
+    endif()
 endif()
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET "netcdf-fortran")
 
+if(CABLE_TESTS)
+    enable_testing()
+    include(FetchContent)
+    if(CABLE_MPI)
+        option(FORTUNO_WITH_MPI "Fortuno: whether to build the MPI interface" ON)
+        FetchContent_Declare(
+            FortunoMPI
+            GIT_REPOSITORY https://github.com/fortuno-repos/fortuno
+            GIT_TAG main
+        )
+        FetchContent_MakeAvailable(FortunoMPI)
+        set(fortuno_libs Fortuno::fortuno_mpi)
+    else()
+        option(FORTUNO_WITH_MPI "Fortuno: whether to build the MPI interface" OFF)
+        FetchContent_Declare(Fortuno
+            GIT_REPOSITORY https://github.com/fortuno-repos/fortuno
+            GIT_TAG main
+            FIND_PACKAGE_ARGS CONFIG
+        )
+        FetchContent_MakeAvailable(Fortuno)
+        set(fortuno_libs Fortuno::fortuno_serial)
+    endif ()
+endif()
+
 set(CABLE_Intel_Fortran_FLAGS -fp-model precise)
 set(CABLE_Intel_Fortran_FLAGS_DEBUG -O0 -g -traceback -fpe0)
 set(CABLE_Intel_Fortran_FLAGS_RELEASE -O2)
@@ -239,6 +268,7 @@ else()
         src/util/cable_common.F90
         src/shared/casa_offline_inout.F90
         src/shared/casa_ncdf.F90
+        src/offline/cable_io_decomp.F90
         src/offline/cable_iovars.F90
         src/offline/cable_surface_types.F90
         src/offline/cable_define_types.F90
@@ -271,13 +301,26 @@ else()
         src/offline/spincasacnp.F90
         src/util/cable_climate_type_mod.F90
         src/util/masks_cbl.F90
+        src/util/cable_array_utils.F90
+        src/util/netcdf/cable_netcdf_decomp_util.F90
+        src/util/netcdf/cable_netcdf.F90
+        src/util/netcdf/cable_netcdf_internal.F90
+        src/util/netcdf/cable_netcdf_stub_types.F90
+        src/util/netcdf/nf90/cable_netcdf_nf90.F90
     )
 
     target_link_libraries(cable_common PRIVATE PkgConfig::NETCDF)
 
     if(CABLE_MPI)
-    target_compile_definitions(cable_common PRIVATE __MPI__)
-    target_link_libraries(cable_common PRIVATE MPI::MPI_Fortran)
+        target_compile_definitions(cable_common PRIVATE __MPI__)
+        target_link_libraries(cable_common PRIVATE MPI::MPI_Fortran)
+    endif()
+
+    if(TARGET PIO::PIO_Fortran)
+        target_link_libraries(cable_common PRIVATE PIO::PIO_Fortran)
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio.F90)
+    else()
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio_stub.F90)
     endif()
 
     if(CABLE_MPI)
@@ -302,4 +345,33 @@ else()
         install(TARGETS cable RUNTIME)
     endif()
 
+    if (CABLE_TESTS)
+        add_executable(
+            cable-tests
+            tests/cable_tests.F90
+            tests/fixtures.F90
+            tests/utils/file_utils.F90
+            tests/test_cable_netcdf.F90
+        )
+        if(CABLE_MPI)
+            target_sources(cable-tests PRIVATE tests/fortuno_interface_mpi.f90)
+        else()
+            target_sources(cable-tests PRIVATE tests/fortuno_interface_serial.f90)
+        endif()
+        target_link_libraries(cable-tests PRIVATE cable_common ${fortuno_libs})
+        if(CABLE_MPI)
+            add_test(NAME cable-tests-serial
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:cable-tests> ~parallel
+            )
+            set_tests_properties(cable-tests-serial PROPERTIES PROCESSORS 1)
+            add_test(NAME cable-tests-parallel
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 $<TARGET_FILE:cable-tests> parallel
+            )
+            set_tests_properties(cable-tests-parallel PROPERTIES PROCESSORS 4)
+        else()
+            add_test(NAME cable-tests-serial
+                COMMAND $<TARGET_FILE:cable-tests> ~parallel
+            )
+        endif()
+    endif()
 endif()

build.bash

@@ -17,6 +17,9 @@ options below will be passed to CMake when generating the build system.
 Options:
   -c, --clean   Delete build directory before invoking CMake.
   -m, --mpi     Compile MPI executable.
+  -p, --parallelio
+                Enable parallel I/O support. This flag requires that --mpi is
+                also set.
   -C, --compiler <compiler>
                 Specify the compiler to use.
   -n, --ncpus <ncpus>
@@ -63,6 +66,9 @@ while [ ${#} -gt 0 ]; do
             mpi=1
             cmake_args+=(-DCABLE_MPI="ON")
             ;;
+        -p|--parallelio)
+            pio=1
+            ;;
         -l|--library)
             build_args+=(--target cable_science)
             cmake_args+=(-DCABLE_LIBRARY="ON")
@@ -98,9 +104,15 @@ if hostname -f | grep gadi.nci.org.au > /dev/null; then
     module add netcdf/4.6.3
     case ${compiler} in
         intel)
-            module add intel-compiler/2019.5.281
+            module add intel-compiler-llvm/2025.0.4
             compiler_lib_install_dir=Intel
-            [[ -n ${mpi} ]] && module add intel-mpi/2019.5.281
+            [[ -n ${mpi} ]] && module add openmpi/4.1.7
+            # This is required so that the Parallel IO library is discoverable
+            # via CMake's `find_package` mechanism:
+            # TODO(Sean): This install of Parallel IO is specific to
+            # openmpi/4.1.7. We need a better way to provide this library on
+            # Gadi.
+            [[ -n ${pio} ]] && prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/parallelio_install"
             ;;
         gnu)
             module add gcc/13.2.0
@@ -123,6 +135,15 @@ if hostname -f | grep gadi.nci.org.au > /dev/null; then
         prepend_path CMAKE_PREFIX_PATH "${OPENMPI_BASE}/include/${compiler_lib_install_dir}"
     fi
 
+    if [[ -n ${pio} ]]; then
+        # The NetCDF Fortran version must be consistent with the version used in Parallel IO
+        # TODO(Sean): we need a better way to provide these libraries on Gadi
+        prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-c-4.9.2-oxepdmgcx6raxo4vi4teu45qqr63v3uj"
+        prepend_path PKG_CONFIG_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-c-4.9.2-oxepdmgcx6raxo4vi4teu45qqr63v3uj/lib/pkgconfig"
+        prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-fortran-4.6.1-eq777uogbelnhv43ln6jyub2gbmos42x"
+        prepend_path PKG_CONFIG_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-fortran-4.6.1-eq777uogbelnhv43ln6jyub2gbmos42x/lib/pkgconfig"
+    fi
+
 elif hostname -f | grep -E '(mc16|mcmini)' > /dev/null; then
     : "${compiler:=gnu}"
 

src/offline/cable_abort.F90

@@ -20,55 +20,38 @@
 
 MODULE cable_abort_module
 
+  USE iso_fortran_env, ONLY: error_unit
   USE cable_IO_vars_module, ONLY: check, logn
-  IMPLICIT NONE
-
-CONTAINS
+  USE cable_mpi_mod, ONLY: mpi_grp_t
 
-  !==============================================================================
-  !
-  ! Name: abort
-  !
-  ! Purpose: Prints an error message and stops the code
-  !
-  ! CALLed from: get_default_inits
-  !              get_restart_data
-  !              get_default_lai
-  !              open_met_file
-  !              get_met_data
-  !              load_parameters
-  !              open_output_file
-  !              write_output
-  !              read_gridinfo
-  !              countpatch
-  !              get_type_parameters
-  !              readpar_i
-  !              readpar_r
-  !              readpar_rd
-  !              readpar_r2
-  !              readpar_r2d
-  !              define_output_variable_r1
-  !              define_output_variable_r2
-  !              define_output_parameter_r1
-  !              define_output_parameter_r2
-  !              write_output_variable_r1
-  !              write_output_variable_r2
-  !              write_output_parameter_r1
-  !              write_output_parameter_r1d
-  !              write_output_parameter_r2
-  !              write_output_parameter_r2d
-  !
-  !==============================================================================
+  IMPLICIT NONE
 
-  SUBROUTINE abort(message)
+  TYPE(mpi_grp_t), PRIVATE :: mpi_grp_global
 
-    ! Input arguments
-    CHARACTER(LEN=*), INTENT(IN) :: message
+CONTAINS
 
-    WRITE (*, *) message
-    STOP 1
+  SUBROUTINE cable_abort_module_init(mpi_grp)
+    !! Initialise abort module
+    TYPE(mpi_grp_t), intent(in) :: mpi_grp
+    mpi_grp_global = mpi_grp
+  END SUBROUTINE
+
+  SUBROUTINE cable_abort(message, file, line)
+    !! Print the error message and stop the code
+    CHARACTER(LEN=*), INTENT(IN) :: message !! Error message
+    CHARACTER(LEN=*), INTENT(IN), OPTIONAL :: file
+    INTEGER, INTENT(IN), OPTIONAL :: line
+    CHARACTER(5) :: line_string
+
+    IF (present(file) .AND. present(line)) THEN
+      WRITE (line_string, "(I5)") line
+      WRITE (error_unit, *) file // ":" // trim(adjustl(line_string)) // ": " // message
+    ELSE
+      WRITE (error_unit, *) message
+    END IF
+    call mpi_grp_global%abort()
 
-  END SUBROUTINE abort
+  END SUBROUTINE
 
   !==============================================================================
   !

src/offline/cable_define_types.F90

@@ -37,16 +37,19 @@ MODULE cable_def_types_mod
   !---CABLE default KINDs for representing INTEGER/REAL values
   !---at least 10-digit precision
 
-  INTEGER :: mp,    & ! # total no of patches/tiles
-       mvtype,& ! total # vegetation types,   from input
+  INTEGER :: mp           !! Total number of patches/tiles in the local grid of this MPI rank
+  INTEGER :: mp_global    !! Total number of patches/tiles in the global grid
+  INTEGER :: mvtype       !! Total number of vegetation types
 #ifdef UM_BUILD 
-       mstype=9,& ! total # soil types, needs to be defined at compile time for now
+  INTEGER :: mstype = 9   !! Total number of soil types, needs to be defined at compile time for now
 #else       
-       mstype,& ! total # soil types,         from input
+  INTEGER :: mstype       !! Total number of soil types, from input
 #endif
-       mland,& !                          ! # land grid cells
-       mpatch  !number of patches   ! mpatch added by rk4417 - phase2
-               !allows for setting this to a const value
+  INTEGER :: mland        !! Total number of land grid cells in the local grid of this MPI rank
+  INTEGER :: mland_global !! Total number of land grid cells in the global grid
+  INTEGER :: mpatch ! mpatch added by rk4417 - phase2
+     !! Number of patches - allows for setting this to a const value
+
 
   INTEGER, PARAMETER ::                                                        &
        i_d  = KIND(9), &

src/offline/cable_driver_common.F90

@@ -42,6 +42,7 @@ MODULE cable_driver_common_mod
   USE CABLE_PLUME_MIP, ONLY : PLUME_MIP_TYPE, PLUME_MIP_INIT
   USE CABLE_CRU, ONLY : CRU_TYPE, CRU_INIT
   USE CABLE_site, ONLY : site_TYPE, site_INIT
+  USE cable_abort_module, ONLY : cable_abort_module_init
   IMPLICIT NONE
   PRIVATE
 
@@ -122,6 +123,8 @@ SUBROUTINE cable_driver_init(mpi_grp, NRRRR)
     INTEGER :: ioerror, unit
     CHARACTER(len=4) :: cRank ! for worker-logfiles
 
+    CALL cable_abort_module_init(mpi_grp)
+
     !check to see if first argument passed to cable is
     !the name of the namelist file
     !if not use cable.nml

src/offline/cable_initialise.F90

@@ -31,7 +31,7 @@
 
 MODULE cable_init_module
 
-  USE cable_abort_module,       ONLY: abort, nc_abort
+  USE cable_abort_module,       ONLY: cable_abort, nc_abort
   USE cable_def_types_mod
   USE cable_IO_vars_module,       ONLY: latitude,longitude, patch,            &
        landpt,smoy,ncid_rin,max_vegpatches,          &
@@ -58,7 +58,7 @@ MODULE cable_init_module
   !
   ! CALLed from: load_parameters
   !
-  ! CALLs: abort
+  ! CALLs: cable_abort
   !
   !==============================================================================
 
@@ -120,8 +120,8 @@ SUBROUTINE get_default_inits(met,soil,ssnow,canopy,logn, EMSOIL)
 
     END DO
 
-    IF(ANY(ssnow%tgg>350.0).OR.ANY(ssnow%tgg<180.0)) CALL abort('Soil temps nuts')
-    IF(ANY(ssnow%albsoilsn>1.0).OR.ANY(ssnow%albsoilsn<0.0)) CALL abort('Albedo nuts')
+    IF(ANY(ssnow%tgg>350.0).OR.ANY(ssnow%tgg<180.0)) CALL cable_abort('Soil temps nuts')
+    IF(ANY(ssnow%albsoilsn>1.0).OR.ANY(ssnow%albsoilsn<0.0)) CALL cable_abort('Albedo nuts')
 
     ! Site independent initialisations (all gridcells):
     ! soil+snow albedo for infrared (other values read in below):
@@ -154,7 +154,7 @@ END SUBROUTINE get_default_inits
   ! CALLs: nc_abort
   !        extraRestart
   !        readpar
-  !        abort
+  !        cable_abort
   !
   ! Input file: [restart].nc
   !
@@ -220,7 +220,7 @@ SUBROUTINE get_restart_data(logn,ssnow,canopy,rough,bgc,                       &
     IF(ok /= NF90_NOERR) CALL nc_abort                                          &
          (ok,'Error finding number of land points in restart file '             &
          //TRIM(filename%restart_in)//' (SUBROUTINE get_restart)')
-    IF(mland_restart /= mland) CALL abort('Number of land points in '//         &
+    IF(mland_restart /= mland) CALL cable_abort('Number of land points in '//         &
          'restart file '//TRIM(filename%restart_in)//                           &
          ' differs from number in met file '//TRIM(filename%met))
 
@@ -252,7 +252,7 @@ SUBROUTINE get_restart_data(logn,ssnow,canopy,rough,bgc,                       &
     !         (ok,'Error finding number of surface types in restart file ' &
     !         //TRIM(filename%restart_in)//' (SUBROUTINE get_restart)')
     !    IF(surftype_restart /= 4) CALL &
-    !         abort('Number of surface types per grid cell in '// &
+    !         cable_abort('Number of surface types per grid cell in '// &
     !         'restart file '//TRIM(filename%restart_in)// &
     !         ' differs from number in cable_variables.f90 ')
     !    ! Get surffrac variable:
@@ -295,11 +295,11 @@ SUBROUTINE get_restart_data(logn,ssnow,canopy,rough,bgc,                       &
     IF(ok/=NF90_NOERR) CALL nc_abort(ok,'Error reading longitude in file '      &
          //TRIM(filename%restart_in)// '(SUBROUTINE get_restart)')
     IF(ANY(ABS(lat_restart-latitude)>0.01))                                     &
-         CALL abort('Latitude of land points in '//                             &
+         CALL cable_abort('Latitude of land points in '//                             &
          'restart file '//TRIM(filename%restart_in)//                           &
          ' differs from met file '//TRIM(filename%met))
     IF(ANY(ABS(lon_restart-longitude)>0.01))                                    &
-         CALL abort('Longitude of land points in '//                            &
+         CALL cable_abort('Longitude of land points in '//                            &
          'restart file '//TRIM(filename%restart_in)//                           &
          ' differs from met file '//TRIM(filename%met))
     DEALLOCATE(lat_restart,lon_restart)
@@ -496,7 +496,7 @@ SUBROUTINE get_restart_data(logn,ssnow,canopy,rough,bgc,                       &
        IF (ANY(INvar /= veg%iveg)) THEN
           PRINT *, 'Error: veg type in restart file different from met input'
           PRINT *, 'Recommend not using this restart file as parameters have changed.'
-          CALL abort('Check iveg in '//filename%restart_in)
+          CALL cable_abort('Check iveg in '//filename%restart_in)
        ENDIF
     ELSE
        ! no problem with overwriting default values
@@ -539,7 +539,7 @@ SUBROUTINE get_restart_data(logn,ssnow,canopy,rough,bgc,                       &
        IF (ANY(INvar /= soil%isoilm)) THEN
           PRINT *, 'Error: soil type in restart file different from met input'
           PRINT *, 'Recommend not using this restart file as parameters have changed.'
-          CALL abort('Check isoil in '//filename%restart_in)
+          CALL cable_abort('Check isoil in '//filename%restart_in)
        ENDIF
     ELSE
        ! no problem with overwriting default values